aboutsummaryrefslogtreecommitdiffstats
path: root/arch/powerpc/kvm
diff options
context:
space:
mode:
authorPaul Mackerras <paulus@samba.org>2011-06-28 20:17:58 -0400
committerAvi Kivity <avi@redhat.com>2011-07-12 06:16:47 -0400
commitf05ed4d56e9cff1c46d2b3049ba0c72e7e29392f (patch)
tree1e31d94a782229cf363779645247de49b0d7a76d /arch/powerpc/kvm
parentc4befc58a0cc5a8cc5b4a7234d67b6b16dec4e70 (diff)
KVM: PPC: Split out code from book3s.c into book3s_pr.c
In preparation for adding code to enable KVM to use hypervisor mode on 64-bit Book 3S processors, this splits book3s.c into two files, book3s.c and book3s_pr.c, where book3s_pr.c contains the code that is specific to running the guest in problem state (user mode) and book3s.c contains code which should apply to all Book 3S processors. In doing this, we abstract some details, namely the interrupt offset, updating the interrupt pending flag, and detecting if the guest is in a critical section. These are all things that will be different when we use hypervisor mode. Signed-off-by: Paul Mackerras <paulus@samba.org> Signed-off-by: Alexander Graf <agraf@suse.de>
Diffstat (limited to 'arch/powerpc/kvm')
-rw-r--r--arch/powerpc/kvm/Makefile2
-rw-r--r--arch/powerpc/kvm/book3s.c995
-rw-r--r--arch/powerpc/kvm/book3s_pr.c1009
3 files changed, 1018 insertions, 988 deletions
diff --git a/arch/powerpc/kvm/Makefile b/arch/powerpc/kvm/Makefile
index 4d6863823f69..bf9854fa7f63 100644
--- a/arch/powerpc/kvm/Makefile
+++ b/arch/powerpc/kvm/Makefile
@@ -43,6 +43,7 @@ kvm-book3s_64-objs := \
43 fpu.o \ 43 fpu.o \
44 book3s_paired_singles.o \ 44 book3s_paired_singles.o \
45 book3s.o \ 45 book3s.o \
46 book3s_pr.o \
46 book3s_emulate.o \ 47 book3s_emulate.o \
47 book3s_interrupts.o \ 48 book3s_interrupts.o \
48 book3s_mmu_hpte.o \ 49 book3s_mmu_hpte.o \
@@ -56,6 +57,7 @@ kvm-book3s_32-objs := \
56 fpu.o \ 57 fpu.o \
57 book3s_paired_singles.o \ 58 book3s_paired_singles.o \
58 book3s.o \ 59 book3s.o \
60 book3s_pr.o \
59 book3s_emulate.o \ 61 book3s_emulate.o \
60 book3s_interrupts.o \ 62 book3s_interrupts.o \
61 book3s_mmu_hpte.o \ 63 book3s_mmu_hpte.o \
diff --git a/arch/powerpc/kvm/book3s.c b/arch/powerpc/kvm/book3s.c
index 5d0babefe913..163e3e13c87c 100644
--- a/arch/powerpc/kvm/book3s.c
+++ b/arch/powerpc/kvm/book3s.c
@@ -38,17 +38,6 @@
38#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU 38#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
39 39
40/* #define EXIT_DEBUG */ 40/* #define EXIT_DEBUG */
41/* #define DEBUG_EXT */
42
43static int kvmppc_handle_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr,
44 ulong msr);
45
46/* Some compatibility defines */
47#ifdef CONFIG_PPC_BOOK3S_32
48#define MSR_USER32 MSR_USER
49#define MSR_USER64 MSR_USER
50#define HW_PAGE_SIZE PAGE_SIZE
51#endif
52 41
53struct kvm_stats_debugfs_item debugfs_entries[] = { 42struct kvm_stats_debugfs_item debugfs_entries[] = {
54 { "exits", VCPU_STAT(sum_exits) }, 43 { "exits", VCPU_STAT(sum_exits) },
@@ -79,100 +68,11 @@ void kvmppc_core_load_guest_debugstate(struct kvm_vcpu *vcpu)
79{ 68{
80} 69}
81 70
82void kvmppc_core_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
83{
84#ifdef CONFIG_PPC_BOOK3S_64
85 memcpy(to_svcpu(vcpu)->slb, to_book3s(vcpu)->slb_shadow, sizeof(to_svcpu(vcpu)->slb));
86 memcpy(&get_paca()->shadow_vcpu, to_book3s(vcpu)->shadow_vcpu,
87 sizeof(get_paca()->shadow_vcpu));
88 to_svcpu(vcpu)->slb_max = to_book3s(vcpu)->slb_shadow_max;
89#endif
90
91#ifdef CONFIG_PPC_BOOK3S_32
92 current->thread.kvm_shadow_vcpu = to_book3s(vcpu)->shadow_vcpu;
93#endif
94}
95
96void kvmppc_core_vcpu_put(struct kvm_vcpu *vcpu)
97{
98#ifdef CONFIG_PPC_BOOK3S_64
99 memcpy(to_book3s(vcpu)->slb_shadow, to_svcpu(vcpu)->slb, sizeof(to_svcpu(vcpu)->slb));
100 memcpy(to_book3s(vcpu)->shadow_vcpu, &get_paca()->shadow_vcpu,
101 sizeof(get_paca()->shadow_vcpu));
102 to_book3s(vcpu)->slb_shadow_max = to_svcpu(vcpu)->slb_max;
103#endif
104
105 kvmppc_giveup_ext(vcpu, MSR_FP);
106 kvmppc_giveup_ext(vcpu, MSR_VEC);
107 kvmppc_giveup_ext(vcpu, MSR_VSX);
108}
109
110static void kvmppc_recalc_shadow_msr(struct kvm_vcpu *vcpu)
111{
112 ulong smsr = vcpu->arch.shared->msr;
113
114 /* Guest MSR values */
115 smsr &= MSR_FE0 | MSR_FE1 | MSR_SF | MSR_SE | MSR_BE | MSR_DE;
116 /* Process MSR values */
117 smsr |= MSR_ME | MSR_RI | MSR_IR | MSR_DR | MSR_PR | MSR_EE;
118 /* External providers the guest reserved */
119 smsr |= (vcpu->arch.shared->msr & vcpu->arch.guest_owned_ext);
120 /* 64-bit Process MSR values */
121#ifdef CONFIG_PPC_BOOK3S_64
122 smsr |= MSR_ISF | MSR_HV;
123#endif
124 vcpu->arch.shadow_msr = smsr;
125}
126
127void kvmppc_set_msr(struct kvm_vcpu *vcpu, u64 msr)
128{
129 ulong old_msr = vcpu->arch.shared->msr;
130
131#ifdef EXIT_DEBUG
132 printk(KERN_INFO "KVM: Set MSR to 0x%llx\n", msr);
133#endif
134
135 msr &= to_book3s(vcpu)->msr_mask;
136 vcpu->arch.shared->msr = msr;
137 kvmppc_recalc_shadow_msr(vcpu);
138
139 if (msr & MSR_POW) {
140 if (!vcpu->arch.pending_exceptions) {
141 kvm_vcpu_block(vcpu);
142 vcpu->stat.halt_wakeup++;
143
144 /* Unset POW bit after we woke up */
145 msr &= ~MSR_POW;
146 vcpu->arch.shared->msr = msr;
147 }
148 }
149
150 if ((vcpu->arch.shared->msr & (MSR_PR|MSR_IR|MSR_DR)) !=
151 (old_msr & (MSR_PR|MSR_IR|MSR_DR))) {
152 kvmppc_mmu_flush_segments(vcpu);
153 kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu));
154
155 /* Preload magic page segment when in kernel mode */
156 if (!(msr & MSR_PR) && vcpu->arch.magic_page_pa) {
157 struct kvm_vcpu_arch *a = &vcpu->arch;
158
159 if (msr & MSR_DR)
160 kvmppc_mmu_map_segment(vcpu, a->magic_page_ea);
161 else
162 kvmppc_mmu_map_segment(vcpu, a->magic_page_pa);
163 }
164 }
165
166 /* Preload FPU if it's enabled */
167 if (vcpu->arch.shared->msr & MSR_FP)
168 kvmppc_handle_ext(vcpu, BOOK3S_INTERRUPT_FP_UNAVAIL, MSR_FP);
169}
170
171void kvmppc_inject_interrupt(struct kvm_vcpu *vcpu, int vec, u64 flags) 71void kvmppc_inject_interrupt(struct kvm_vcpu *vcpu, int vec, u64 flags)
172{ 72{
173 vcpu->arch.shared->srr0 = kvmppc_get_pc(vcpu); 73 vcpu->arch.shared->srr0 = kvmppc_get_pc(vcpu);
174 vcpu->arch.shared->srr1 = vcpu->arch.shared->msr | flags; 74 vcpu->arch.shared->srr1 = vcpu->arch.shared->msr | flags;
175 kvmppc_set_pc(vcpu, to_book3s(vcpu)->hior + vec); 75 kvmppc_set_pc(vcpu, kvmppc_interrupt_offset(vcpu) + vec);
176 vcpu->arch.mmu.reset_msr(vcpu); 76 vcpu->arch.mmu.reset_msr(vcpu);
177} 77}
178 78
@@ -206,11 +106,13 @@ static int kvmppc_book3s_vec2irqprio(unsigned int vec)
206static void kvmppc_book3s_dequeue_irqprio(struct kvm_vcpu *vcpu, 106static void kvmppc_book3s_dequeue_irqprio(struct kvm_vcpu *vcpu,
207 unsigned int vec) 107 unsigned int vec)
208{ 108{
109 unsigned long old_pending = vcpu->arch.pending_exceptions;
110
209 clear_bit(kvmppc_book3s_vec2irqprio(vec), 111 clear_bit(kvmppc_book3s_vec2irqprio(vec),
210 &vcpu->arch.pending_exceptions); 112 &vcpu->arch.pending_exceptions);
211 113
212 if (!vcpu->arch.pending_exceptions) 114 kvmppc_update_int_pending(vcpu, vcpu->arch.pending_exceptions,
213 vcpu->arch.shared->int_pending = 0; 115 old_pending);
214} 116}
215 117
216void kvmppc_book3s_queue_irqprio(struct kvm_vcpu *vcpu, unsigned int vec) 118void kvmppc_book3s_queue_irqprio(struct kvm_vcpu *vcpu, unsigned int vec)
@@ -269,20 +171,7 @@ int kvmppc_book3s_irqprio_deliver(struct kvm_vcpu *vcpu, unsigned int priority)
269 int deliver = 1; 171 int deliver = 1;
270 int vec = 0; 172 int vec = 0;
271 ulong flags = 0ULL; 173 ulong flags = 0ULL;
272 ulong crit_raw = vcpu->arch.shared->critical; 174 bool crit = kvmppc_critical_section(vcpu);
273 ulong crit_r1 = kvmppc_get_gpr(vcpu, 1);
274 bool crit;
275
276 /* Truncate crit indicators in 32 bit mode */
277 if (!(vcpu->arch.shared->msr & MSR_SF)) {
278 crit_raw &= 0xffffffff;
279 crit_r1 &= 0xffffffff;
280 }
281
282 /* Critical section when crit == r1 */
283 crit = (crit_raw == crit_r1);
284 /* ... and we're in supervisor mode */
285 crit = crit && !(vcpu->arch.shared->msr & MSR_PR);
286 175
287 switch (priority) { 176 switch (priority) {
288 case BOOK3S_IRQPRIO_DECREMENTER: 177 case BOOK3S_IRQPRIO_DECREMENTER:
@@ -394,64 +283,7 @@ void kvmppc_core_deliver_interrupts(struct kvm_vcpu *vcpu)
394 } 283 }
395 284
396 /* Tell the guest about our interrupt status */ 285 /* Tell the guest about our interrupt status */
397 if (*pending) 286 kvmppc_update_int_pending(vcpu, *pending, old_pending);
398 vcpu->arch.shared->int_pending = 1;
399 else if (old_pending)
400 vcpu->arch.shared->int_pending = 0;
401}
402
403void kvmppc_set_pvr(struct kvm_vcpu *vcpu, u32 pvr)
404{
405 u32 host_pvr;
406
407 vcpu->arch.hflags &= ~BOOK3S_HFLAG_SLB;
408 vcpu->arch.pvr = pvr;
409#ifdef CONFIG_PPC_BOOK3S_64
410 if ((pvr >= 0x330000) && (pvr < 0x70330000)) {
411 kvmppc_mmu_book3s_64_init(vcpu);
412 to_book3s(vcpu)->hior = 0xfff00000;
413 to_book3s(vcpu)->msr_mask = 0xffffffffffffffffULL;
414 } else
415#endif
416 {
417 kvmppc_mmu_book3s_32_init(vcpu);
418 to_book3s(vcpu)->hior = 0;
419 to_book3s(vcpu)->msr_mask = 0xffffffffULL;
420 }
421
422 /* If we are in hypervisor level on 970, we can tell the CPU to
423 * treat DCBZ as 32 bytes store */
424 vcpu->arch.hflags &= ~BOOK3S_HFLAG_DCBZ32;
425 if (vcpu->arch.mmu.is_dcbz32(vcpu) && (mfmsr() & MSR_HV) &&
426 !strcmp(cur_cpu_spec->platform, "ppc970"))
427 vcpu->arch.hflags |= BOOK3S_HFLAG_DCBZ32;
428
429 /* Cell performs badly if MSR_FEx are set. So let's hope nobody
430 really needs them in a VM on Cell and force disable them. */
431 if (!strcmp(cur_cpu_spec->platform, "ppc-cell-be"))
432 to_book3s(vcpu)->msr_mask &= ~(MSR_FE0 | MSR_FE1);
433
434#ifdef CONFIG_PPC_BOOK3S_32
435 /* 32 bit Book3S always has 32 byte dcbz */
436 vcpu->arch.hflags |= BOOK3S_HFLAG_DCBZ32;
437#endif
438
439 /* On some CPUs we can execute paired single operations natively */
440 asm ( "mfpvr %0" : "=r"(host_pvr));
441 switch (host_pvr) {
442 case 0x00080200: /* lonestar 2.0 */
443 case 0x00088202: /* lonestar 2.2 */
444 case 0x70000100: /* gekko 1.0 */
445 case 0x00080100: /* gekko 2.0 */
446 case 0x00083203: /* gekko 2.3a */
447 case 0x00083213: /* gekko 2.3b */
448 case 0x00083204: /* gekko 2.4 */
449 case 0x00083214: /* gekko 2.4e (8SE) - retail HW2 */
450 case 0x00087200: /* broadway */
451 vcpu->arch.hflags |= BOOK3S_HFLAG_NATIVE_PS;
452 /* Enable HID2.PSE - in case we need it later */
453 mtspr(SPRN_HID2_GEKKO, mfspr(SPRN_HID2_GEKKO) | (1 << 29));
454 }
455} 287}
456 288
457pfn_t kvmppc_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn) 289pfn_t kvmppc_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn)
@@ -473,44 +305,6 @@ pfn_t kvmppc_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn)
473 return gfn_to_pfn(vcpu->kvm, gfn); 305 return gfn_to_pfn(vcpu->kvm, gfn);
474} 306}
475 307
476/* Book3s_32 CPUs always have 32 bytes cache line size, which Linux assumes. To
477 * make Book3s_32 Linux work on Book3s_64, we have to make sure we trap dcbz to
478 * emulate 32 bytes dcbz length.
479 *
480 * The Book3s_64 inventors also realized this case and implemented a special bit
481 * in the HID5 register, which is a hypervisor ressource. Thus we can't use it.
482 *
483 * My approach here is to patch the dcbz instruction on executing pages.
484 */
485static void kvmppc_patch_dcbz(struct kvm_vcpu *vcpu, struct kvmppc_pte *pte)
486{
487 struct page *hpage;
488 u64 hpage_offset;
489 u32 *page;
490 int i;
491
492 hpage = gfn_to_page(vcpu->kvm, pte->raddr >> PAGE_SHIFT);
493 if (is_error_page(hpage)) {
494 kvm_release_page_clean(hpage);
495 return;
496 }
497
498 hpage_offset = pte->raddr & ~PAGE_MASK;
499 hpage_offset &= ~0xFFFULL;
500 hpage_offset /= 4;
501
502 get_page(hpage);
503 page = kmap_atomic(hpage, KM_USER0);
504
505 /* patch dcbz into reserved instruction, so we trap */
506 for (i=hpage_offset; i < hpage_offset + (HW_PAGE_SIZE / 4); i++)
507 if ((page[i] & 0xff0007ff) == INS_DCBZ)
508 page[i] &= 0xfffffff7;
509
510 kunmap_atomic(page, KM_USER0);
511 put_page(hpage);
512}
513
514static int kvmppc_xlate(struct kvm_vcpu *vcpu, ulong eaddr, bool data, 308static int kvmppc_xlate(struct kvm_vcpu *vcpu, ulong eaddr, bool data,
515 struct kvmppc_pte *pte) 309 struct kvmppc_pte *pte)
516{ 310{
@@ -608,519 +402,6 @@ mmio:
608 return EMULATE_DO_MMIO; 402 return EMULATE_DO_MMIO;
609} 403}
610 404
611static int kvmppc_visible_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
612{
613 ulong mp_pa = vcpu->arch.magic_page_pa;
614
615 if (unlikely(mp_pa) &&
616 unlikely((mp_pa & KVM_PAM) >> PAGE_SHIFT == gfn)) {
617 return 1;
618 }
619
620 return kvm_is_visible_gfn(vcpu->kvm, gfn);
621}
622
623int kvmppc_handle_pagefault(struct kvm_run *run, struct kvm_vcpu *vcpu,
624 ulong eaddr, int vec)
625{
626 bool data = (vec == BOOK3S_INTERRUPT_DATA_STORAGE);
627 int r = RESUME_GUEST;
628 int relocated;
629 int page_found = 0;
630 struct kvmppc_pte pte;
631 bool is_mmio = false;
632 bool dr = (vcpu->arch.shared->msr & MSR_DR) ? true : false;
633 bool ir = (vcpu->arch.shared->msr & MSR_IR) ? true : false;
634 u64 vsid;
635
636 relocated = data ? dr : ir;
637
638 /* Resolve real address if translation turned on */
639 if (relocated) {
640 page_found = vcpu->arch.mmu.xlate(vcpu, eaddr, &pte, data);
641 } else {
642 pte.may_execute = true;
643 pte.may_read = true;
644 pte.may_write = true;
645 pte.raddr = eaddr & KVM_PAM;
646 pte.eaddr = eaddr;
647 pte.vpage = eaddr >> 12;
648 }
649
650 switch (vcpu->arch.shared->msr & (MSR_DR|MSR_IR)) {
651 case 0:
652 pte.vpage |= ((u64)VSID_REAL << (SID_SHIFT - 12));
653 break;
654 case MSR_DR:
655 case MSR_IR:
656 vcpu->arch.mmu.esid_to_vsid(vcpu, eaddr >> SID_SHIFT, &vsid);
657
658 if ((vcpu->arch.shared->msr & (MSR_DR|MSR_IR)) == MSR_DR)
659 pte.vpage |= ((u64)VSID_REAL_DR << (SID_SHIFT - 12));
660 else
661 pte.vpage |= ((u64)VSID_REAL_IR << (SID_SHIFT - 12));
662 pte.vpage |= vsid;
663
664 if (vsid == -1)
665 page_found = -EINVAL;
666 break;
667 }
668
669 if (vcpu->arch.mmu.is_dcbz32(vcpu) &&
670 (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32))) {
671 /*
672 * If we do the dcbz hack, we have to NX on every execution,
673 * so we can patch the executing code. This renders our guest
674 * NX-less.
675 */
676 pte.may_execute = !data;
677 }
678
679 if (page_found == -ENOENT) {
680 /* Page not found in guest PTE entries */
681 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu);
682 vcpu->arch.shared->dsisr = to_svcpu(vcpu)->fault_dsisr;
683 vcpu->arch.shared->msr |=
684 (to_svcpu(vcpu)->shadow_srr1 & 0x00000000f8000000ULL);
685 kvmppc_book3s_queue_irqprio(vcpu, vec);
686 } else if (page_found == -EPERM) {
687 /* Storage protection */
688 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu);
689 vcpu->arch.shared->dsisr =
690 to_svcpu(vcpu)->fault_dsisr & ~DSISR_NOHPTE;
691 vcpu->arch.shared->dsisr |= DSISR_PROTFAULT;
692 vcpu->arch.shared->msr |=
693 (to_svcpu(vcpu)->shadow_srr1 & 0x00000000f8000000ULL);
694 kvmppc_book3s_queue_irqprio(vcpu, vec);
695 } else if (page_found == -EINVAL) {
696 /* Page not found in guest SLB */
697 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu);
698 kvmppc_book3s_queue_irqprio(vcpu, vec + 0x80);
699 } else if (!is_mmio &&
700 kvmppc_visible_gfn(vcpu, pte.raddr >> PAGE_SHIFT)) {
701 /* The guest's PTE is not mapped yet. Map on the host */
702 kvmppc_mmu_map_page(vcpu, &pte);
703 if (data)
704 vcpu->stat.sp_storage++;
705 else if (vcpu->arch.mmu.is_dcbz32(vcpu) &&
706 (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32)))
707 kvmppc_patch_dcbz(vcpu, &pte);
708 } else {
709 /* MMIO */
710 vcpu->stat.mmio_exits++;
711 vcpu->arch.paddr_accessed = pte.raddr;
712 r = kvmppc_emulate_mmio(run, vcpu);
713 if ( r == RESUME_HOST_NV )
714 r = RESUME_HOST;
715 }
716
717 return r;
718}
719
720static inline int get_fpr_index(int i)
721{
722#ifdef CONFIG_VSX
723 i *= 2;
724#endif
725 return i;
726}
727
728/* Give up external provider (FPU, Altivec, VSX) */
729void kvmppc_giveup_ext(struct kvm_vcpu *vcpu, ulong msr)
730{
731 struct thread_struct *t = &current->thread;
732 u64 *vcpu_fpr = vcpu->arch.fpr;
733#ifdef CONFIG_VSX
734 u64 *vcpu_vsx = vcpu->arch.vsr;
735#endif
736 u64 *thread_fpr = (u64*)t->fpr;
737 int i;
738
739 if (!(vcpu->arch.guest_owned_ext & msr))
740 return;
741
742#ifdef DEBUG_EXT
743 printk(KERN_INFO "Giving up ext 0x%lx\n", msr);
744#endif
745
746 switch (msr) {
747 case MSR_FP:
748 giveup_fpu(current);
749 for (i = 0; i < ARRAY_SIZE(vcpu->arch.fpr); i++)
750 vcpu_fpr[i] = thread_fpr[get_fpr_index(i)];
751
752 vcpu->arch.fpscr = t->fpscr.val;
753 break;
754 case MSR_VEC:
755#ifdef CONFIG_ALTIVEC
756 giveup_altivec(current);
757 memcpy(vcpu->arch.vr, t->vr, sizeof(vcpu->arch.vr));
758 vcpu->arch.vscr = t->vscr;
759#endif
760 break;
761 case MSR_VSX:
762#ifdef CONFIG_VSX
763 __giveup_vsx(current);
764 for (i = 0; i < ARRAY_SIZE(vcpu->arch.vsr); i++)
765 vcpu_vsx[i] = thread_fpr[get_fpr_index(i) + 1];
766#endif
767 break;
768 default:
769 BUG();
770 }
771
772 vcpu->arch.guest_owned_ext &= ~msr;
773 current->thread.regs->msr &= ~msr;
774 kvmppc_recalc_shadow_msr(vcpu);
775}
776
777static int kvmppc_read_inst(struct kvm_vcpu *vcpu)
778{
779 ulong srr0 = kvmppc_get_pc(vcpu);
780 u32 last_inst = kvmppc_get_last_inst(vcpu);
781 int ret;
782
783 ret = kvmppc_ld(vcpu, &srr0, sizeof(u32), &last_inst, false);
784 if (ret == -ENOENT) {
785 ulong msr = vcpu->arch.shared->msr;
786
787 msr = kvmppc_set_field(msr, 33, 33, 1);
788 msr = kvmppc_set_field(msr, 34, 36, 0);
789 vcpu->arch.shared->msr = kvmppc_set_field(msr, 42, 47, 0);
790 kvmppc_book3s_queue_irqprio(vcpu, BOOK3S_INTERRUPT_INST_STORAGE);
791 return EMULATE_AGAIN;
792 }
793
794 return EMULATE_DONE;
795}
796
797static int kvmppc_check_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr)
798{
799
800 /* Need to do paired single emulation? */
801 if (!(vcpu->arch.hflags & BOOK3S_HFLAG_PAIRED_SINGLE))
802 return EMULATE_DONE;
803
804 /* Read out the instruction */
805 if (kvmppc_read_inst(vcpu) == EMULATE_DONE)
806 /* Need to emulate */
807 return EMULATE_FAIL;
808
809 return EMULATE_AGAIN;
810}
811
812/* Handle external providers (FPU, Altivec, VSX) */
813static int kvmppc_handle_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr,
814 ulong msr)
815{
816 struct thread_struct *t = &current->thread;
817 u64 *vcpu_fpr = vcpu->arch.fpr;
818#ifdef CONFIG_VSX
819 u64 *vcpu_vsx = vcpu->arch.vsr;
820#endif
821 u64 *thread_fpr = (u64*)t->fpr;
822 int i;
823
824 /* When we have paired singles, we emulate in software */
825 if (vcpu->arch.hflags & BOOK3S_HFLAG_PAIRED_SINGLE)
826 return RESUME_GUEST;
827
828 if (!(vcpu->arch.shared->msr & msr)) {
829 kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
830 return RESUME_GUEST;
831 }
832
833 /* We already own the ext */
834 if (vcpu->arch.guest_owned_ext & msr) {
835 return RESUME_GUEST;
836 }
837
838#ifdef DEBUG_EXT
839 printk(KERN_INFO "Loading up ext 0x%lx\n", msr);
840#endif
841
842 current->thread.regs->msr |= msr;
843
844 switch (msr) {
845 case MSR_FP:
846 for (i = 0; i < ARRAY_SIZE(vcpu->arch.fpr); i++)
847 thread_fpr[get_fpr_index(i)] = vcpu_fpr[i];
848
849 t->fpscr.val = vcpu->arch.fpscr;
850 t->fpexc_mode = 0;
851 kvmppc_load_up_fpu();
852 break;
853 case MSR_VEC:
854#ifdef CONFIG_ALTIVEC
855 memcpy(t->vr, vcpu->arch.vr, sizeof(vcpu->arch.vr));
856 t->vscr = vcpu->arch.vscr;
857 t->vrsave = -1;
858 kvmppc_load_up_altivec();
859#endif
860 break;
861 case MSR_VSX:
862#ifdef CONFIG_VSX
863 for (i = 0; i < ARRAY_SIZE(vcpu->arch.vsr); i++)
864 thread_fpr[get_fpr_index(i) + 1] = vcpu_vsx[i];
865 kvmppc_load_up_vsx();
866#endif
867 break;
868 default:
869 BUG();
870 }
871
872 vcpu->arch.guest_owned_ext |= msr;
873
874 kvmppc_recalc_shadow_msr(vcpu);
875
876 return RESUME_GUEST;
877}
878
879int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu,
880 unsigned int exit_nr)
881{
882 int r = RESUME_HOST;
883
884 vcpu->stat.sum_exits++;
885
886 run->exit_reason = KVM_EXIT_UNKNOWN;
887 run->ready_for_interrupt_injection = 1;
888
889 trace_kvm_book3s_exit(exit_nr, vcpu);
890 kvm_resched(vcpu);
891 switch (exit_nr) {
892 case BOOK3S_INTERRUPT_INST_STORAGE:
893 vcpu->stat.pf_instruc++;
894
895#ifdef CONFIG_PPC_BOOK3S_32
896 /* We set segments as unused segments when invalidating them. So
897 * treat the respective fault as segment fault. */
898 if (to_svcpu(vcpu)->sr[kvmppc_get_pc(vcpu) >> SID_SHIFT]
899 == SR_INVALID) {
900 kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu));
901 r = RESUME_GUEST;
902 break;
903 }
904#endif
905
906 /* only care about PTEG not found errors, but leave NX alone */
907 if (to_svcpu(vcpu)->shadow_srr1 & 0x40000000) {
908 r = kvmppc_handle_pagefault(run, vcpu, kvmppc_get_pc(vcpu), exit_nr);
909 vcpu->stat.sp_instruc++;
910 } else if (vcpu->arch.mmu.is_dcbz32(vcpu) &&
911 (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32))) {
912 /*
913 * XXX If we do the dcbz hack we use the NX bit to flush&patch the page,
914 * so we can't use the NX bit inside the guest. Let's cross our fingers,
915 * that no guest that needs the dcbz hack does NX.
916 */
917 kvmppc_mmu_pte_flush(vcpu, kvmppc_get_pc(vcpu), ~0xFFFUL);
918 r = RESUME_GUEST;
919 } else {
920 vcpu->arch.shared->msr |=
921 to_svcpu(vcpu)->shadow_srr1 & 0x58000000;
922 kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
923 r = RESUME_GUEST;
924 }
925 break;
926 case BOOK3S_INTERRUPT_DATA_STORAGE:
927 {
928 ulong dar = kvmppc_get_fault_dar(vcpu);
929 vcpu->stat.pf_storage++;
930
931#ifdef CONFIG_PPC_BOOK3S_32
932 /* We set segments as unused segments when invalidating them. So
933 * treat the respective fault as segment fault. */
934 if ((to_svcpu(vcpu)->sr[dar >> SID_SHIFT]) == SR_INVALID) {
935 kvmppc_mmu_map_segment(vcpu, dar);
936 r = RESUME_GUEST;
937 break;
938 }
939#endif
940
941 /* The only case we need to handle is missing shadow PTEs */
942 if (to_svcpu(vcpu)->fault_dsisr & DSISR_NOHPTE) {
943 r = kvmppc_handle_pagefault(run, vcpu, dar, exit_nr);
944 } else {
945 vcpu->arch.shared->dar = dar;
946 vcpu->arch.shared->dsisr = to_svcpu(vcpu)->fault_dsisr;
947 kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
948 r = RESUME_GUEST;
949 }
950 break;
951 }
952 case BOOK3S_INTERRUPT_DATA_SEGMENT:
953 if (kvmppc_mmu_map_segment(vcpu, kvmppc_get_fault_dar(vcpu)) < 0) {
954 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu);
955 kvmppc_book3s_queue_irqprio(vcpu,
956 BOOK3S_INTERRUPT_DATA_SEGMENT);
957 }
958 r = RESUME_GUEST;
959 break;
960 case BOOK3S_INTERRUPT_INST_SEGMENT:
961 if (kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu)) < 0) {
962 kvmppc_book3s_queue_irqprio(vcpu,
963 BOOK3S_INTERRUPT_INST_SEGMENT);
964 }
965 r = RESUME_GUEST;
966 break;
967 /* We're good on these - the host merely wanted to get our attention */
968 case BOOK3S_INTERRUPT_DECREMENTER:
969 vcpu->stat.dec_exits++;
970 r = RESUME_GUEST;
971 break;
972 case BOOK3S_INTERRUPT_EXTERNAL:
973 vcpu->stat.ext_intr_exits++;
974 r = RESUME_GUEST;
975 break;
976 case BOOK3S_INTERRUPT_PERFMON:
977 r = RESUME_GUEST;
978 break;
979 case BOOK3S_INTERRUPT_PROGRAM:
980 {
981 enum emulation_result er;
982 ulong flags;
983
984program_interrupt:
985 flags = to_svcpu(vcpu)->shadow_srr1 & 0x1f0000ull;
986
987 if (vcpu->arch.shared->msr & MSR_PR) {
988#ifdef EXIT_DEBUG
989 printk(KERN_INFO "Userspace triggered 0x700 exception at 0x%lx (0x%x)\n", kvmppc_get_pc(vcpu), kvmppc_get_last_inst(vcpu));
990#endif
991 if ((kvmppc_get_last_inst(vcpu) & 0xff0007ff) !=
992 (INS_DCBZ & 0xfffffff7)) {
993 kvmppc_core_queue_program(vcpu, flags);
994 r = RESUME_GUEST;
995 break;
996 }
997 }
998
999 vcpu->stat.emulated_inst_exits++;
1000 er = kvmppc_emulate_instruction(run, vcpu);
1001 switch (er) {
1002 case EMULATE_DONE:
1003 r = RESUME_GUEST_NV;
1004 break;
1005 case EMULATE_AGAIN:
1006 r = RESUME_GUEST;
1007 break;
1008 case EMULATE_FAIL:
1009 printk(KERN_CRIT "%s: emulation at %lx failed (%08x)\n",
1010 __func__, kvmppc_get_pc(vcpu), kvmppc_get_last_inst(vcpu));
1011 kvmppc_core_queue_program(vcpu, flags);
1012 r = RESUME_GUEST;
1013 break;
1014 case EMULATE_DO_MMIO:
1015 run->exit_reason = KVM_EXIT_MMIO;
1016 r = RESUME_HOST_NV;
1017 break;
1018 default:
1019 BUG();
1020 }
1021 break;
1022 }
1023 case BOOK3S_INTERRUPT_SYSCALL:
1024 if (vcpu->arch.osi_enabled &&
1025 (((u32)kvmppc_get_gpr(vcpu, 3)) == OSI_SC_MAGIC_R3) &&
1026 (((u32)kvmppc_get_gpr(vcpu, 4)) == OSI_SC_MAGIC_R4)) {
1027 /* MOL hypercalls */
1028 u64 *gprs = run->osi.gprs;
1029 int i;
1030
1031 run->exit_reason = KVM_EXIT_OSI;
1032 for (i = 0; i < 32; i++)
1033 gprs[i] = kvmppc_get_gpr(vcpu, i);
1034 vcpu->arch.osi_needed = 1;
1035 r = RESUME_HOST_NV;
1036 } else if (!(vcpu->arch.shared->msr & MSR_PR) &&
1037 (((u32)kvmppc_get_gpr(vcpu, 0)) == KVM_SC_MAGIC_R0)) {
1038 /* KVM PV hypercalls */
1039 kvmppc_set_gpr(vcpu, 3, kvmppc_kvm_pv(vcpu));
1040 r = RESUME_GUEST;
1041 } else {
1042 /* Guest syscalls */
1043 vcpu->stat.syscall_exits++;
1044 kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
1045 r = RESUME_GUEST;
1046 }
1047 break;
1048 case BOOK3S_INTERRUPT_FP_UNAVAIL:
1049 case BOOK3S_INTERRUPT_ALTIVEC:
1050 case BOOK3S_INTERRUPT_VSX:
1051 {
1052 int ext_msr = 0;
1053
1054 switch (exit_nr) {
1055 case BOOK3S_INTERRUPT_FP_UNAVAIL: ext_msr = MSR_FP; break;
1056 case BOOK3S_INTERRUPT_ALTIVEC: ext_msr = MSR_VEC; break;
1057 case BOOK3S_INTERRUPT_VSX: ext_msr = MSR_VSX; break;
1058 }
1059
1060 switch (kvmppc_check_ext(vcpu, exit_nr)) {
1061 case EMULATE_DONE:
1062 /* everything ok - let's enable the ext */
1063 r = kvmppc_handle_ext(vcpu, exit_nr, ext_msr);
1064 break;
1065 case EMULATE_FAIL:
1066 /* we need to emulate this instruction */
1067 goto program_interrupt;
1068 break;
1069 default:
1070 /* nothing to worry about - go again */
1071 break;
1072 }
1073 break;
1074 }
1075 case BOOK3S_INTERRUPT_ALIGNMENT:
1076 if (kvmppc_read_inst(vcpu) == EMULATE_DONE) {
1077 vcpu->arch.shared->dsisr = kvmppc_alignment_dsisr(vcpu,
1078 kvmppc_get_last_inst(vcpu));
1079 vcpu->arch.shared->dar = kvmppc_alignment_dar(vcpu,
1080 kvmppc_get_last_inst(vcpu));
1081 kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
1082 }
1083 r = RESUME_GUEST;
1084 break;
1085 case BOOK3S_INTERRUPT_MACHINE_CHECK:
1086 case BOOK3S_INTERRUPT_TRACE:
1087 kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
1088 r = RESUME_GUEST;
1089 break;
1090 default:
1091 /* Ugh - bork here! What did we get? */
1092 printk(KERN_EMERG "exit_nr=0x%x | pc=0x%lx | msr=0x%lx\n",
1093 exit_nr, kvmppc_get_pc(vcpu), to_svcpu(vcpu)->shadow_srr1);
1094 r = RESUME_HOST;
1095 BUG();
1096 break;
1097 }
1098
1099
1100 if (!(r & RESUME_HOST)) {
1101 /* To avoid clobbering exit_reason, only check for signals if
1102 * we aren't already exiting to userspace for some other
1103 * reason. */
1104 if (signal_pending(current)) {
1105#ifdef EXIT_DEBUG
1106 printk(KERN_EMERG "KVM: Going back to host\n");
1107#endif
1108 vcpu->stat.signal_exits++;
1109 run->exit_reason = KVM_EXIT_INTR;
1110 r = -EINTR;
1111 } else {
1112 /* In case an interrupt came in that was triggered
1113 * from userspace (like DEC), we need to check what
1114 * to inject now! */
1115 kvmppc_core_deliver_interrupts(vcpu);
1116 }
1117 }
1118
1119 trace_kvm_book3s_reenter(r, vcpu);
1120
1121 return r;
1122}
1123
1124int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) 405int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1125{ 406{
1126 return 0; 407 return 0;
@@ -1181,69 +462,6 @@ int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1181 return 0; 462 return 0;
1182} 463}
1183 464
1184int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1185 struct kvm_sregs *sregs)
1186{
1187 struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu);
1188 int i;
1189
1190 sregs->pvr = vcpu->arch.pvr;
1191
1192 sregs->u.s.sdr1 = to_book3s(vcpu)->sdr1;
1193 if (vcpu->arch.hflags & BOOK3S_HFLAG_SLB) {
1194 for (i = 0; i < 64; i++) {
1195 sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige | i;
1196 sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv;
1197 }
1198 } else {
1199 for (i = 0; i < 16; i++)
1200 sregs->u.s.ppc32.sr[i] = vcpu->arch.shared->sr[i];
1201
1202 for (i = 0; i < 8; i++) {
1203 sregs->u.s.ppc32.ibat[i] = vcpu3s->ibat[i].raw;
1204 sregs->u.s.ppc32.dbat[i] = vcpu3s->dbat[i].raw;
1205 }
1206 }
1207
1208 return 0;
1209}
1210
1211int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1212 struct kvm_sregs *sregs)
1213{
1214 struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu);
1215 int i;
1216
1217 kvmppc_set_pvr(vcpu, sregs->pvr);
1218
1219 vcpu3s->sdr1 = sregs->u.s.sdr1;
1220 if (vcpu->arch.hflags & BOOK3S_HFLAG_SLB) {
1221 for (i = 0; i < 64; i++) {
1222 vcpu->arch.mmu.slbmte(vcpu, sregs->u.s.ppc64.slb[i].slbv,
1223 sregs->u.s.ppc64.slb[i].slbe);
1224 }
1225 } else {
1226 for (i = 0; i < 16; i++) {
1227 vcpu->arch.mmu.mtsrin(vcpu, i, sregs->u.s.ppc32.sr[i]);
1228 }
1229 for (i = 0; i < 8; i++) {
1230 kvmppc_set_bat(vcpu, &(vcpu3s->ibat[i]), false,
1231 (u32)sregs->u.s.ppc32.ibat[i]);
1232 kvmppc_set_bat(vcpu, &(vcpu3s->ibat[i]), true,
1233 (u32)(sregs->u.s.ppc32.ibat[i] >> 32));
1234 kvmppc_set_bat(vcpu, &(vcpu3s->dbat[i]), false,
1235 (u32)sregs->u.s.ppc32.dbat[i]);
1236 kvmppc_set_bat(vcpu, &(vcpu3s->dbat[i]), true,
1237 (u32)(sregs->u.s.ppc32.dbat[i] >> 32));
1238 }
1239 }
1240
1241 /* Flush the MMU after messing with the segments */
1242 kvmppc_mmu_pte_flush(vcpu, 0, 0);
1243
1244 return 0;
1245}
1246
1247int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 465int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1248{ 466{
1249 return -ENOTSUPP; 467 return -ENOTSUPP;
@@ -1298,202 +516,3 @@ out:
1298 mutex_unlock(&kvm->slots_lock); 516 mutex_unlock(&kvm->slots_lock);
1299 return r; 517 return r;
1300} 518}
1301
1302int kvmppc_core_check_processor_compat(void)
1303{
1304 return 0;
1305}
1306
1307struct kvm_vcpu *kvmppc_core_vcpu_create(struct kvm *kvm, unsigned int id)
1308{
1309 struct kvmppc_vcpu_book3s *vcpu_book3s;
1310 struct kvm_vcpu *vcpu;
1311 int err = -ENOMEM;
1312 unsigned long p;
1313
1314 vcpu_book3s = vzalloc(sizeof(struct kvmppc_vcpu_book3s));
1315 if (!vcpu_book3s)
1316 goto out;
1317
1318 vcpu_book3s->shadow_vcpu = (struct kvmppc_book3s_shadow_vcpu *)
1319 kzalloc(sizeof(*vcpu_book3s->shadow_vcpu), GFP_KERNEL);
1320 if (!vcpu_book3s->shadow_vcpu)
1321 goto free_vcpu;
1322
1323 vcpu = &vcpu_book3s->vcpu;
1324 err = kvm_vcpu_init(vcpu, kvm, id);
1325 if (err)
1326 goto free_shadow_vcpu;
1327
1328 p = __get_free_page(GFP_KERNEL|__GFP_ZERO);
1329 /* the real shared page fills the last 4k of our page */
1330 vcpu->arch.shared = (void*)(p + PAGE_SIZE - 4096);
1331 if (!p)
1332 goto uninit_vcpu;
1333
1334 vcpu->arch.host_retip = kvm_return_point;
1335 vcpu->arch.host_msr = mfmsr();
1336#ifdef CONFIG_PPC_BOOK3S_64
1337 /* default to book3s_64 (970fx) */
1338 vcpu->arch.pvr = 0x3C0301;
1339#else
1340 /* default to book3s_32 (750) */
1341 vcpu->arch.pvr = 0x84202;
1342#endif
1343 kvmppc_set_pvr(vcpu, vcpu->arch.pvr);
1344 vcpu->arch.slb_nr = 64;
1345
1346 /* remember where some real-mode handlers are */
1347 vcpu->arch.trampoline_lowmem = __pa(kvmppc_handler_lowmem_trampoline);
1348 vcpu->arch.trampoline_enter = __pa(kvmppc_handler_trampoline_enter);
1349 vcpu->arch.highmem_handler = (ulong)kvmppc_handler_highmem;
1350#ifdef CONFIG_PPC_BOOK3S_64
1351 vcpu->arch.rmcall = *(ulong*)kvmppc_rmcall;
1352#else
1353 vcpu->arch.rmcall = (ulong)kvmppc_rmcall;
1354#endif
1355
1356 vcpu->arch.shadow_msr = MSR_USER64;
1357
1358 err = kvmppc_mmu_init(vcpu);
1359 if (err < 0)
1360 goto uninit_vcpu;
1361
1362 return vcpu;
1363
1364uninit_vcpu:
1365 kvm_vcpu_uninit(vcpu);
1366free_shadow_vcpu:
1367 kfree(vcpu_book3s->shadow_vcpu);
1368free_vcpu:
1369 vfree(vcpu_book3s);
1370out:
1371 return ERR_PTR(err);
1372}
1373
1374void kvmppc_core_vcpu_free(struct kvm_vcpu *vcpu)
1375{
1376 struct kvmppc_vcpu_book3s *vcpu_book3s = to_book3s(vcpu);
1377
1378 free_page((unsigned long)vcpu->arch.shared & PAGE_MASK);
1379 kvm_vcpu_uninit(vcpu);
1380 kfree(vcpu_book3s->shadow_vcpu);
1381 vfree(vcpu_book3s);
1382}
1383
1384extern int __kvmppc_vcpu_entry(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu);
1385int __kvmppc_vcpu_run(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
1386{
1387 int ret;
1388 double fpr[32][TS_FPRWIDTH];
1389 unsigned int fpscr;
1390 int fpexc_mode;
1391#ifdef CONFIG_ALTIVEC
1392 vector128 vr[32];
1393 vector128 vscr;
1394 unsigned long uninitialized_var(vrsave);
1395 int used_vr;
1396#endif
1397#ifdef CONFIG_VSX
1398 int used_vsr;
1399#endif
1400 ulong ext_msr;
1401
1402 /* No need to go into the guest when all we do is going out */
1403 if (signal_pending(current)) {
1404 kvm_run->exit_reason = KVM_EXIT_INTR;
1405 return -EINTR;
1406 }
1407
1408 /* Save FPU state in stack */
1409 if (current->thread.regs->msr & MSR_FP)
1410 giveup_fpu(current);
1411 memcpy(fpr, current->thread.fpr, sizeof(current->thread.fpr));
1412 fpscr = current->thread.fpscr.val;
1413 fpexc_mode = current->thread.fpexc_mode;
1414
1415#ifdef CONFIG_ALTIVEC
1416 /* Save Altivec state in stack */
1417 used_vr = current->thread.used_vr;
1418 if (used_vr) {
1419 if (current->thread.regs->msr & MSR_VEC)
1420 giveup_altivec(current);
1421 memcpy(vr, current->thread.vr, sizeof(current->thread.vr));
1422 vscr = current->thread.vscr;
1423 vrsave = current->thread.vrsave;
1424 }
1425#endif
1426
1427#ifdef CONFIG_VSX
1428 /* Save VSX state in stack */
1429 used_vsr = current->thread.used_vsr;
1430 if (used_vsr && (current->thread.regs->msr & MSR_VSX))
1431 __giveup_vsx(current);
1432#endif
1433
1434 /* Remember the MSR with disabled extensions */
1435 ext_msr = current->thread.regs->msr;
1436
1437 /* XXX we get called with irq disabled - change that! */
1438 local_irq_enable();
1439
1440 /* Preload FPU if it's enabled */
1441 if (vcpu->arch.shared->msr & MSR_FP)
1442 kvmppc_handle_ext(vcpu, BOOK3S_INTERRUPT_FP_UNAVAIL, MSR_FP);
1443
1444 ret = __kvmppc_vcpu_entry(kvm_run, vcpu);
1445
1446 local_irq_disable();
1447
1448 current->thread.regs->msr = ext_msr;
1449
1450 /* Make sure we save the guest FPU/Altivec/VSX state */
1451 kvmppc_giveup_ext(vcpu, MSR_FP);
1452 kvmppc_giveup_ext(vcpu, MSR_VEC);
1453 kvmppc_giveup_ext(vcpu, MSR_VSX);
1454
1455 /* Restore FPU state from stack */
1456 memcpy(current->thread.fpr, fpr, sizeof(current->thread.fpr));
1457 current->thread.fpscr.val = fpscr;
1458 current->thread.fpexc_mode = fpexc_mode;
1459
1460#ifdef CONFIG_ALTIVEC
1461 /* Restore Altivec state from stack */
1462 if (used_vr && current->thread.used_vr) {
1463 memcpy(current->thread.vr, vr, sizeof(current->thread.vr));
1464 current->thread.vscr = vscr;
1465 current->thread.vrsave = vrsave;
1466 }
1467 current->thread.used_vr = used_vr;
1468#endif
1469
1470#ifdef CONFIG_VSX
1471 current->thread.used_vsr = used_vsr;
1472#endif
1473
1474 return ret;
1475}
1476
1477static int kvmppc_book3s_init(void)
1478{
1479 int r;
1480
1481 r = kvm_init(NULL, sizeof(struct kvmppc_vcpu_book3s), 0,
1482 THIS_MODULE);
1483
1484 if (r)
1485 return r;
1486
1487 r = kvmppc_mmu_hpte_sysinit();
1488
1489 return r;
1490}
1491
1492static void kvmppc_book3s_exit(void)
1493{
1494 kvmppc_mmu_hpte_sysexit();
1495 kvm_exit();
1496}
1497
1498module_init(kvmppc_book3s_init);
1499module_exit(kvmppc_book3s_exit);
diff --git a/arch/powerpc/kvm/book3s_pr.c b/arch/powerpc/kvm/book3s_pr.c
new file mode 100644
index 000000000000..fcdc97eb9411
--- /dev/null
+++ b/arch/powerpc/kvm/book3s_pr.c
@@ -0,0 +1,1009 @@
1/*
2 * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved.
3 *
4 * Authors:
5 * Alexander Graf <agraf@suse.de>
6 * Kevin Wolf <mail@kevin-wolf.de>
7 * Paul Mackerras <paulus@samba.org>
8 *
9 * Description:
10 * Functions relating to running KVM on Book 3S processors where
11 * we don't have access to hypervisor mode, and we run the guest
12 * in problem state (user mode).
13 *
14 * This file is derived from arch/powerpc/kvm/44x.c,
15 * by Hollis Blanchard <hollisb@us.ibm.com>.
16 *
17 * This program is free software; you can redistribute it and/or modify
18 * it under the terms of the GNU General Public License, version 2, as
19 * published by the Free Software Foundation.
20 */
21
22#include <linux/kvm_host.h>
23#include <linux/err.h>
24#include <linux/slab.h>
25
26#include <asm/reg.h>
27#include <asm/cputable.h>
28#include <asm/cacheflush.h>
29#include <asm/tlbflush.h>
30#include <asm/uaccess.h>
31#include <asm/io.h>
32#include <asm/kvm_ppc.h>
33#include <asm/kvm_book3s.h>
34#include <asm/mmu_context.h>
35#include <linux/gfp.h>
36#include <linux/sched.h>
37#include <linux/vmalloc.h>
38#include <linux/highmem.h>
39
40#include "trace.h"
41
42/* #define EXIT_DEBUG */
43/* #define DEBUG_EXT */
44
45static int kvmppc_handle_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr,
46 ulong msr);
47
48/* Some compatibility defines */
49#ifdef CONFIG_PPC_BOOK3S_32
50#define MSR_USER32 MSR_USER
51#define MSR_USER64 MSR_USER
52#define HW_PAGE_SIZE PAGE_SIZE
53#endif
54
55void kvmppc_core_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
56{
57#ifdef CONFIG_PPC_BOOK3S_64
58 memcpy(to_svcpu(vcpu)->slb, to_book3s(vcpu)->slb_shadow, sizeof(to_svcpu(vcpu)->slb));
59 memcpy(&get_paca()->shadow_vcpu, to_book3s(vcpu)->shadow_vcpu,
60 sizeof(get_paca()->shadow_vcpu));
61 to_svcpu(vcpu)->slb_max = to_book3s(vcpu)->slb_shadow_max;
62#endif
63
64#ifdef CONFIG_PPC_BOOK3S_32
65 current->thread.kvm_shadow_vcpu = to_book3s(vcpu)->shadow_vcpu;
66#endif
67}
68
69void kvmppc_core_vcpu_put(struct kvm_vcpu *vcpu)
70{
71#ifdef CONFIG_PPC_BOOK3S_64
72 memcpy(to_book3s(vcpu)->slb_shadow, to_svcpu(vcpu)->slb, sizeof(to_svcpu(vcpu)->slb));
73 memcpy(to_book3s(vcpu)->shadow_vcpu, &get_paca()->shadow_vcpu,
74 sizeof(get_paca()->shadow_vcpu));
75 to_book3s(vcpu)->slb_shadow_max = to_svcpu(vcpu)->slb_max;
76#endif
77
78 kvmppc_giveup_ext(vcpu, MSR_FP);
79 kvmppc_giveup_ext(vcpu, MSR_VEC);
80 kvmppc_giveup_ext(vcpu, MSR_VSX);
81}
82
83static void kvmppc_recalc_shadow_msr(struct kvm_vcpu *vcpu)
84{
85 ulong smsr = vcpu->arch.shared->msr;
86
87 /* Guest MSR values */
88 smsr &= MSR_FE0 | MSR_FE1 | MSR_SF | MSR_SE | MSR_BE | MSR_DE;
89 /* Process MSR values */
90 smsr |= MSR_ME | MSR_RI | MSR_IR | MSR_DR | MSR_PR | MSR_EE;
91 /* External providers the guest reserved */
92 smsr |= (vcpu->arch.shared->msr & vcpu->arch.guest_owned_ext);
93 /* 64-bit Process MSR values */
94#ifdef CONFIG_PPC_BOOK3S_64
95 smsr |= MSR_ISF | MSR_HV;
96#endif
97 vcpu->arch.shadow_msr = smsr;
98}
99
100void kvmppc_set_msr(struct kvm_vcpu *vcpu, u64 msr)
101{
102 ulong old_msr = vcpu->arch.shared->msr;
103
104#ifdef EXIT_DEBUG
105 printk(KERN_INFO "KVM: Set MSR to 0x%llx\n", msr);
106#endif
107
108 msr &= to_book3s(vcpu)->msr_mask;
109 vcpu->arch.shared->msr = msr;
110 kvmppc_recalc_shadow_msr(vcpu);
111
112 if (msr & MSR_POW) {
113 if (!vcpu->arch.pending_exceptions) {
114 kvm_vcpu_block(vcpu);
115 vcpu->stat.halt_wakeup++;
116
117 /* Unset POW bit after we woke up */
118 msr &= ~MSR_POW;
119 vcpu->arch.shared->msr = msr;
120 }
121 }
122
123 if ((vcpu->arch.shared->msr & (MSR_PR|MSR_IR|MSR_DR)) !=
124 (old_msr & (MSR_PR|MSR_IR|MSR_DR))) {
125 kvmppc_mmu_flush_segments(vcpu);
126 kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu));
127
128 /* Preload magic page segment when in kernel mode */
129 if (!(msr & MSR_PR) && vcpu->arch.magic_page_pa) {
130 struct kvm_vcpu_arch *a = &vcpu->arch;
131
132 if (msr & MSR_DR)
133 kvmppc_mmu_map_segment(vcpu, a->magic_page_ea);
134 else
135 kvmppc_mmu_map_segment(vcpu, a->magic_page_pa);
136 }
137 }
138
139 /* Preload FPU if it's enabled */
140 if (vcpu->arch.shared->msr & MSR_FP)
141 kvmppc_handle_ext(vcpu, BOOK3S_INTERRUPT_FP_UNAVAIL, MSR_FP);
142}
143
144void kvmppc_set_pvr(struct kvm_vcpu *vcpu, u32 pvr)
145{
146 u32 host_pvr;
147
148 vcpu->arch.hflags &= ~BOOK3S_HFLAG_SLB;
149 vcpu->arch.pvr = pvr;
150#ifdef CONFIG_PPC_BOOK3S_64
151 if ((pvr >= 0x330000) && (pvr < 0x70330000)) {
152 kvmppc_mmu_book3s_64_init(vcpu);
153 to_book3s(vcpu)->hior = 0xfff00000;
154 to_book3s(vcpu)->msr_mask = 0xffffffffffffffffULL;
155 } else
156#endif
157 {
158 kvmppc_mmu_book3s_32_init(vcpu);
159 to_book3s(vcpu)->hior = 0;
160 to_book3s(vcpu)->msr_mask = 0xffffffffULL;
161 }
162
163 /* If we are in hypervisor level on 970, we can tell the CPU to
164 * treat DCBZ as 32 bytes store */
165 vcpu->arch.hflags &= ~BOOK3S_HFLAG_DCBZ32;
166 if (vcpu->arch.mmu.is_dcbz32(vcpu) && (mfmsr() & MSR_HV) &&
167 !strcmp(cur_cpu_spec->platform, "ppc970"))
168 vcpu->arch.hflags |= BOOK3S_HFLAG_DCBZ32;
169
170 /* Cell performs badly if MSR_FEx are set. So let's hope nobody
171 really needs them in a VM on Cell and force disable them. */
172 if (!strcmp(cur_cpu_spec->platform, "ppc-cell-be"))
173 to_book3s(vcpu)->msr_mask &= ~(MSR_FE0 | MSR_FE1);
174
175#ifdef CONFIG_PPC_BOOK3S_32
176 /* 32 bit Book3S always has 32 byte dcbz */
177 vcpu->arch.hflags |= BOOK3S_HFLAG_DCBZ32;
178#endif
179
180 /* On some CPUs we can execute paired single operations natively */
181 asm ( "mfpvr %0" : "=r"(host_pvr));
182 switch (host_pvr) {
183 case 0x00080200: /* lonestar 2.0 */
184 case 0x00088202: /* lonestar 2.2 */
185 case 0x70000100: /* gekko 1.0 */
186 case 0x00080100: /* gekko 2.0 */
187 case 0x00083203: /* gekko 2.3a */
188 case 0x00083213: /* gekko 2.3b */
189 case 0x00083204: /* gekko 2.4 */
190 case 0x00083214: /* gekko 2.4e (8SE) - retail HW2 */
191 case 0x00087200: /* broadway */
192 vcpu->arch.hflags |= BOOK3S_HFLAG_NATIVE_PS;
193 /* Enable HID2.PSE - in case we need it later */
194 mtspr(SPRN_HID2_GEKKO, mfspr(SPRN_HID2_GEKKO) | (1 << 29));
195 }
196}
197
198/* Book3s_32 CPUs always have 32 bytes cache line size, which Linux assumes. To
199 * make Book3s_32 Linux work on Book3s_64, we have to make sure we trap dcbz to
200 * emulate 32 bytes dcbz length.
201 *
202 * The Book3s_64 inventors also realized this case and implemented a special bit
203 * in the HID5 register, which is a hypervisor ressource. Thus we can't use it.
204 *
205 * My approach here is to patch the dcbz instruction on executing pages.
206 */
207static void kvmppc_patch_dcbz(struct kvm_vcpu *vcpu, struct kvmppc_pte *pte)
208{
209 struct page *hpage;
210 u64 hpage_offset;
211 u32 *page;
212 int i;
213
214 hpage = gfn_to_page(vcpu->kvm, pte->raddr >> PAGE_SHIFT);
215 if (is_error_page(hpage)) {
216 kvm_release_page_clean(hpage);
217 return;
218 }
219
220 hpage_offset = pte->raddr & ~PAGE_MASK;
221 hpage_offset &= ~0xFFFULL;
222 hpage_offset /= 4;
223
224 get_page(hpage);
225 page = kmap_atomic(hpage, KM_USER0);
226
227 /* patch dcbz into reserved instruction, so we trap */
228 for (i=hpage_offset; i < hpage_offset + (HW_PAGE_SIZE / 4); i++)
229 if ((page[i] & 0xff0007ff) == INS_DCBZ)
230 page[i] &= 0xfffffff7;
231
232 kunmap_atomic(page, KM_USER0);
233 put_page(hpage);
234}
235
236static int kvmppc_visible_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
237{
238 ulong mp_pa = vcpu->arch.magic_page_pa;
239
240 if (unlikely(mp_pa) &&
241 unlikely((mp_pa & KVM_PAM) >> PAGE_SHIFT == gfn)) {
242 return 1;
243 }
244
245 return kvm_is_visible_gfn(vcpu->kvm, gfn);
246}
247
248int kvmppc_handle_pagefault(struct kvm_run *run, struct kvm_vcpu *vcpu,
249 ulong eaddr, int vec)
250{
251 bool data = (vec == BOOK3S_INTERRUPT_DATA_STORAGE);
252 int r = RESUME_GUEST;
253 int relocated;
254 int page_found = 0;
255 struct kvmppc_pte pte;
256 bool is_mmio = false;
257 bool dr = (vcpu->arch.shared->msr & MSR_DR) ? true : false;
258 bool ir = (vcpu->arch.shared->msr & MSR_IR) ? true : false;
259 u64 vsid;
260
261 relocated = data ? dr : ir;
262
263 /* Resolve real address if translation turned on */
264 if (relocated) {
265 page_found = vcpu->arch.mmu.xlate(vcpu, eaddr, &pte, data);
266 } else {
267 pte.may_execute = true;
268 pte.may_read = true;
269 pte.may_write = true;
270 pte.raddr = eaddr & KVM_PAM;
271 pte.eaddr = eaddr;
272 pte.vpage = eaddr >> 12;
273 }
274
275 switch (vcpu->arch.shared->msr & (MSR_DR|MSR_IR)) {
276 case 0:
277 pte.vpage |= ((u64)VSID_REAL << (SID_SHIFT - 12));
278 break;
279 case MSR_DR:
280 case MSR_IR:
281 vcpu->arch.mmu.esid_to_vsid(vcpu, eaddr >> SID_SHIFT, &vsid);
282
283 if ((vcpu->arch.shared->msr & (MSR_DR|MSR_IR)) == MSR_DR)
284 pte.vpage |= ((u64)VSID_REAL_DR << (SID_SHIFT - 12));
285 else
286 pte.vpage |= ((u64)VSID_REAL_IR << (SID_SHIFT - 12));
287 pte.vpage |= vsid;
288
289 if (vsid == -1)
290 page_found = -EINVAL;
291 break;
292 }
293
294 if (vcpu->arch.mmu.is_dcbz32(vcpu) &&
295 (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32))) {
296 /*
297 * If we do the dcbz hack, we have to NX on every execution,
298 * so we can patch the executing code. This renders our guest
299 * NX-less.
300 */
301 pte.may_execute = !data;
302 }
303
304 if (page_found == -ENOENT) {
305 /* Page not found in guest PTE entries */
306 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu);
307 vcpu->arch.shared->dsisr = to_svcpu(vcpu)->fault_dsisr;
308 vcpu->arch.shared->msr |=
309 (to_svcpu(vcpu)->shadow_srr1 & 0x00000000f8000000ULL);
310 kvmppc_book3s_queue_irqprio(vcpu, vec);
311 } else if (page_found == -EPERM) {
312 /* Storage protection */
313 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu);
314 vcpu->arch.shared->dsisr =
315 to_svcpu(vcpu)->fault_dsisr & ~DSISR_NOHPTE;
316 vcpu->arch.shared->dsisr |= DSISR_PROTFAULT;
317 vcpu->arch.shared->msr |=
318 (to_svcpu(vcpu)->shadow_srr1 & 0x00000000f8000000ULL);
319 kvmppc_book3s_queue_irqprio(vcpu, vec);
320 } else if (page_found == -EINVAL) {
321 /* Page not found in guest SLB */
322 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu);
323 kvmppc_book3s_queue_irqprio(vcpu, vec + 0x80);
324 } else if (!is_mmio &&
325 kvmppc_visible_gfn(vcpu, pte.raddr >> PAGE_SHIFT)) {
326 /* The guest's PTE is not mapped yet. Map on the host */
327 kvmppc_mmu_map_page(vcpu, &pte);
328 if (data)
329 vcpu->stat.sp_storage++;
330 else if (vcpu->arch.mmu.is_dcbz32(vcpu) &&
331 (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32)))
332 kvmppc_patch_dcbz(vcpu, &pte);
333 } else {
334 /* MMIO */
335 vcpu->stat.mmio_exits++;
336 vcpu->arch.paddr_accessed = pte.raddr;
337 r = kvmppc_emulate_mmio(run, vcpu);
338 if ( r == RESUME_HOST_NV )
339 r = RESUME_HOST;
340 }
341
342 return r;
343}
344
345static inline int get_fpr_index(int i)
346{
347#ifdef CONFIG_VSX
348 i *= 2;
349#endif
350 return i;
351}
352
353/* Give up external provider (FPU, Altivec, VSX) */
354void kvmppc_giveup_ext(struct kvm_vcpu *vcpu, ulong msr)
355{
356 struct thread_struct *t = &current->thread;
357 u64 *vcpu_fpr = vcpu->arch.fpr;
358#ifdef CONFIG_VSX
359 u64 *vcpu_vsx = vcpu->arch.vsr;
360#endif
361 u64 *thread_fpr = (u64*)t->fpr;
362 int i;
363
364 if (!(vcpu->arch.guest_owned_ext & msr))
365 return;
366
367#ifdef DEBUG_EXT
368 printk(KERN_INFO "Giving up ext 0x%lx\n", msr);
369#endif
370
371 switch (msr) {
372 case MSR_FP:
373 giveup_fpu(current);
374 for (i = 0; i < ARRAY_SIZE(vcpu->arch.fpr); i++)
375 vcpu_fpr[i] = thread_fpr[get_fpr_index(i)];
376
377 vcpu->arch.fpscr = t->fpscr.val;
378 break;
379 case MSR_VEC:
380#ifdef CONFIG_ALTIVEC
381 giveup_altivec(current);
382 memcpy(vcpu->arch.vr, t->vr, sizeof(vcpu->arch.vr));
383 vcpu->arch.vscr = t->vscr;
384#endif
385 break;
386 case MSR_VSX:
387#ifdef CONFIG_VSX
388 __giveup_vsx(current);
389 for (i = 0; i < ARRAY_SIZE(vcpu->arch.vsr); i++)
390 vcpu_vsx[i] = thread_fpr[get_fpr_index(i) + 1];
391#endif
392 break;
393 default:
394 BUG();
395 }
396
397 vcpu->arch.guest_owned_ext &= ~msr;
398 current->thread.regs->msr &= ~msr;
399 kvmppc_recalc_shadow_msr(vcpu);
400}
401
402static int kvmppc_read_inst(struct kvm_vcpu *vcpu)
403{
404 ulong srr0 = kvmppc_get_pc(vcpu);
405 u32 last_inst = kvmppc_get_last_inst(vcpu);
406 int ret;
407
408 ret = kvmppc_ld(vcpu, &srr0, sizeof(u32), &last_inst, false);
409 if (ret == -ENOENT) {
410 ulong msr = vcpu->arch.shared->msr;
411
412 msr = kvmppc_set_field(msr, 33, 33, 1);
413 msr = kvmppc_set_field(msr, 34, 36, 0);
414 vcpu->arch.shared->msr = kvmppc_set_field(msr, 42, 47, 0);
415 kvmppc_book3s_queue_irqprio(vcpu, BOOK3S_INTERRUPT_INST_STORAGE);
416 return EMULATE_AGAIN;
417 }
418
419 return EMULATE_DONE;
420}
421
422static int kvmppc_check_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr)
423{
424
425 /* Need to do paired single emulation? */
426 if (!(vcpu->arch.hflags & BOOK3S_HFLAG_PAIRED_SINGLE))
427 return EMULATE_DONE;
428
429 /* Read out the instruction */
430 if (kvmppc_read_inst(vcpu) == EMULATE_DONE)
431 /* Need to emulate */
432 return EMULATE_FAIL;
433
434 return EMULATE_AGAIN;
435}
436
437/* Handle external providers (FPU, Altivec, VSX) */
438static int kvmppc_handle_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr,
439 ulong msr)
440{
441 struct thread_struct *t = &current->thread;
442 u64 *vcpu_fpr = vcpu->arch.fpr;
443#ifdef CONFIG_VSX
444 u64 *vcpu_vsx = vcpu->arch.vsr;
445#endif
446 u64 *thread_fpr = (u64*)t->fpr;
447 int i;
448
449 /* When we have paired singles, we emulate in software */
450 if (vcpu->arch.hflags & BOOK3S_HFLAG_PAIRED_SINGLE)
451 return RESUME_GUEST;
452
453 if (!(vcpu->arch.shared->msr & msr)) {
454 kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
455 return RESUME_GUEST;
456 }
457
458 /* We already own the ext */
459 if (vcpu->arch.guest_owned_ext & msr) {
460 return RESUME_GUEST;
461 }
462
463#ifdef DEBUG_EXT
464 printk(KERN_INFO "Loading up ext 0x%lx\n", msr);
465#endif
466
467 current->thread.regs->msr |= msr;
468
469 switch (msr) {
470 case MSR_FP:
471 for (i = 0; i < ARRAY_SIZE(vcpu->arch.fpr); i++)
472 thread_fpr[get_fpr_index(i)] = vcpu_fpr[i];
473
474 t->fpscr.val = vcpu->arch.fpscr;
475 t->fpexc_mode = 0;
476 kvmppc_load_up_fpu();
477 break;
478 case MSR_VEC:
479#ifdef CONFIG_ALTIVEC
480 memcpy(t->vr, vcpu->arch.vr, sizeof(vcpu->arch.vr));
481 t->vscr = vcpu->arch.vscr;
482 t->vrsave = -1;
483 kvmppc_load_up_altivec();
484#endif
485 break;
486 case MSR_VSX:
487#ifdef CONFIG_VSX
488 for (i = 0; i < ARRAY_SIZE(vcpu->arch.vsr); i++)
489 thread_fpr[get_fpr_index(i) + 1] = vcpu_vsx[i];
490 kvmppc_load_up_vsx();
491#endif
492 break;
493 default:
494 BUG();
495 }
496
497 vcpu->arch.guest_owned_ext |= msr;
498
499 kvmppc_recalc_shadow_msr(vcpu);
500
501 return RESUME_GUEST;
502}
503
504int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu,
505 unsigned int exit_nr)
506{
507 int r = RESUME_HOST;
508
509 vcpu->stat.sum_exits++;
510
511 run->exit_reason = KVM_EXIT_UNKNOWN;
512 run->ready_for_interrupt_injection = 1;
513
514 trace_kvm_book3s_exit(exit_nr, vcpu);
515 kvm_resched(vcpu);
516 switch (exit_nr) {
517 case BOOK3S_INTERRUPT_INST_STORAGE:
518 vcpu->stat.pf_instruc++;
519
520#ifdef CONFIG_PPC_BOOK3S_32
521 /* We set segments as unused segments when invalidating them. So
522 * treat the respective fault as segment fault. */
523 if (to_svcpu(vcpu)->sr[kvmppc_get_pc(vcpu) >> SID_SHIFT]
524 == SR_INVALID) {
525 kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu));
526 r = RESUME_GUEST;
527 break;
528 }
529#endif
530
531 /* only care about PTEG not found errors, but leave NX alone */
532 if (to_svcpu(vcpu)->shadow_srr1 & 0x40000000) {
533 r = kvmppc_handle_pagefault(run, vcpu, kvmppc_get_pc(vcpu), exit_nr);
534 vcpu->stat.sp_instruc++;
535 } else if (vcpu->arch.mmu.is_dcbz32(vcpu) &&
536 (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32))) {
537 /*
538 * XXX If we do the dcbz hack we use the NX bit to flush&patch the page,
539 * so we can't use the NX bit inside the guest. Let's cross our fingers,
540 * that no guest that needs the dcbz hack does NX.
541 */
542 kvmppc_mmu_pte_flush(vcpu, kvmppc_get_pc(vcpu), ~0xFFFUL);
543 r = RESUME_GUEST;
544 } else {
545 vcpu->arch.shared->msr |=
546 to_svcpu(vcpu)->shadow_srr1 & 0x58000000;
547 kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
548 r = RESUME_GUEST;
549 }
550 break;
551 case BOOK3S_INTERRUPT_DATA_STORAGE:
552 {
553 ulong dar = kvmppc_get_fault_dar(vcpu);
554 vcpu->stat.pf_storage++;
555
556#ifdef CONFIG_PPC_BOOK3S_32
557 /* We set segments as unused segments when invalidating them. So
558 * treat the respective fault as segment fault. */
559 if ((to_svcpu(vcpu)->sr[dar >> SID_SHIFT]) == SR_INVALID) {
560 kvmppc_mmu_map_segment(vcpu, dar);
561 r = RESUME_GUEST;
562 break;
563 }
564#endif
565
566 /* The only case we need to handle is missing shadow PTEs */
567 if (to_svcpu(vcpu)->fault_dsisr & DSISR_NOHPTE) {
568 r = kvmppc_handle_pagefault(run, vcpu, dar, exit_nr);
569 } else {
570 vcpu->arch.shared->dar = dar;
571 vcpu->arch.shared->dsisr = to_svcpu(vcpu)->fault_dsisr;
572 kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
573 r = RESUME_GUEST;
574 }
575 break;
576 }
577 case BOOK3S_INTERRUPT_DATA_SEGMENT:
578 if (kvmppc_mmu_map_segment(vcpu, kvmppc_get_fault_dar(vcpu)) < 0) {
579 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu);
580 kvmppc_book3s_queue_irqprio(vcpu,
581 BOOK3S_INTERRUPT_DATA_SEGMENT);
582 }
583 r = RESUME_GUEST;
584 break;
585 case BOOK3S_INTERRUPT_INST_SEGMENT:
586 if (kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu)) < 0) {
587 kvmppc_book3s_queue_irqprio(vcpu,
588 BOOK3S_INTERRUPT_INST_SEGMENT);
589 }
590 r = RESUME_GUEST;
591 break;
592 /* We're good on these - the host merely wanted to get our attention */
593 case BOOK3S_INTERRUPT_DECREMENTER:
594 vcpu->stat.dec_exits++;
595 r = RESUME_GUEST;
596 break;
597 case BOOK3S_INTERRUPT_EXTERNAL:
598 vcpu->stat.ext_intr_exits++;
599 r = RESUME_GUEST;
600 break;
601 case BOOK3S_INTERRUPT_PERFMON:
602 r = RESUME_GUEST;
603 break;
604 case BOOK3S_INTERRUPT_PROGRAM:
605 {
606 enum emulation_result er;
607 ulong flags;
608
609program_interrupt:
610 flags = to_svcpu(vcpu)->shadow_srr1 & 0x1f0000ull;
611
612 if (vcpu->arch.shared->msr & MSR_PR) {
613#ifdef EXIT_DEBUG
614 printk(KERN_INFO "Userspace triggered 0x700 exception at 0x%lx (0x%x)\n", kvmppc_get_pc(vcpu), kvmppc_get_last_inst(vcpu));
615#endif
616 if ((kvmppc_get_last_inst(vcpu) & 0xff0007ff) !=
617 (INS_DCBZ & 0xfffffff7)) {
618 kvmppc_core_queue_program(vcpu, flags);
619 r = RESUME_GUEST;
620 break;
621 }
622 }
623
624 vcpu->stat.emulated_inst_exits++;
625 er = kvmppc_emulate_instruction(run, vcpu);
626 switch (er) {
627 case EMULATE_DONE:
628 r = RESUME_GUEST_NV;
629 break;
630 case EMULATE_AGAIN:
631 r = RESUME_GUEST;
632 break;
633 case EMULATE_FAIL:
634 printk(KERN_CRIT "%s: emulation at %lx failed (%08x)\n",
635 __func__, kvmppc_get_pc(vcpu), kvmppc_get_last_inst(vcpu));
636 kvmppc_core_queue_program(vcpu, flags);
637 r = RESUME_GUEST;
638 break;
639 case EMULATE_DO_MMIO:
640 run->exit_reason = KVM_EXIT_MMIO;
641 r = RESUME_HOST_NV;
642 break;
643 default:
644 BUG();
645 }
646 break;
647 }
648 case BOOK3S_INTERRUPT_SYSCALL:
649 if (vcpu->arch.osi_enabled &&
650 (((u32)kvmppc_get_gpr(vcpu, 3)) == OSI_SC_MAGIC_R3) &&
651 (((u32)kvmppc_get_gpr(vcpu, 4)) == OSI_SC_MAGIC_R4)) {
652 /* MOL hypercalls */
653 u64 *gprs = run->osi.gprs;
654 int i;
655
656 run->exit_reason = KVM_EXIT_OSI;
657 for (i = 0; i < 32; i++)
658 gprs[i] = kvmppc_get_gpr(vcpu, i);
659 vcpu->arch.osi_needed = 1;
660 r = RESUME_HOST_NV;
661 } else if (!(vcpu->arch.shared->msr & MSR_PR) &&
662 (((u32)kvmppc_get_gpr(vcpu, 0)) == KVM_SC_MAGIC_R0)) {
663 /* KVM PV hypercalls */
664 kvmppc_set_gpr(vcpu, 3, kvmppc_kvm_pv(vcpu));
665 r = RESUME_GUEST;
666 } else {
667 /* Guest syscalls */
668 vcpu->stat.syscall_exits++;
669 kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
670 r = RESUME_GUEST;
671 }
672 break;
673 case BOOK3S_INTERRUPT_FP_UNAVAIL:
674 case BOOK3S_INTERRUPT_ALTIVEC:
675 case BOOK3S_INTERRUPT_VSX:
676 {
677 int ext_msr = 0;
678
679 switch (exit_nr) {
680 case BOOK3S_INTERRUPT_FP_UNAVAIL: ext_msr = MSR_FP; break;
681 case BOOK3S_INTERRUPT_ALTIVEC: ext_msr = MSR_VEC; break;
682 case BOOK3S_INTERRUPT_VSX: ext_msr = MSR_VSX; break;
683 }
684
685 switch (kvmppc_check_ext(vcpu, exit_nr)) {
686 case EMULATE_DONE:
687 /* everything ok - let's enable the ext */
688 r = kvmppc_handle_ext(vcpu, exit_nr, ext_msr);
689 break;
690 case EMULATE_FAIL:
691 /* we need to emulate this instruction */
692 goto program_interrupt;
693 break;
694 default:
695 /* nothing to worry about - go again */
696 break;
697 }
698 break;
699 }
700 case BOOK3S_INTERRUPT_ALIGNMENT:
701 if (kvmppc_read_inst(vcpu) == EMULATE_DONE) {
702 vcpu->arch.shared->dsisr = kvmppc_alignment_dsisr(vcpu,
703 kvmppc_get_last_inst(vcpu));
704 vcpu->arch.shared->dar = kvmppc_alignment_dar(vcpu,
705 kvmppc_get_last_inst(vcpu));
706 kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
707 }
708 r = RESUME_GUEST;
709 break;
710 case BOOK3S_INTERRUPT_MACHINE_CHECK:
711 case BOOK3S_INTERRUPT_TRACE:
712 kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
713 r = RESUME_GUEST;
714 break;
715 default:
716 /* Ugh - bork here! What did we get? */
717 printk(KERN_EMERG "exit_nr=0x%x | pc=0x%lx | msr=0x%lx\n",
718 exit_nr, kvmppc_get_pc(vcpu), to_svcpu(vcpu)->shadow_srr1);
719 r = RESUME_HOST;
720 BUG();
721 break;
722 }
723
724
725 if (!(r & RESUME_HOST)) {
726 /* To avoid clobbering exit_reason, only check for signals if
727 * we aren't already exiting to userspace for some other
728 * reason. */
729 if (signal_pending(current)) {
730#ifdef EXIT_DEBUG
731 printk(KERN_EMERG "KVM: Going back to host\n");
732#endif
733 vcpu->stat.signal_exits++;
734 run->exit_reason = KVM_EXIT_INTR;
735 r = -EINTR;
736 } else {
737 /* In case an interrupt came in that was triggered
738 * from userspace (like DEC), we need to check what
739 * to inject now! */
740 kvmppc_core_deliver_interrupts(vcpu);
741 }
742 }
743
744 trace_kvm_book3s_reenter(r, vcpu);
745
746 return r;
747}
748
749int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
750 struct kvm_sregs *sregs)
751{
752 struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu);
753 int i;
754
755 sregs->pvr = vcpu->arch.pvr;
756
757 sregs->u.s.sdr1 = to_book3s(vcpu)->sdr1;
758 if (vcpu->arch.hflags & BOOK3S_HFLAG_SLB) {
759 for (i = 0; i < 64; i++) {
760 sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige | i;
761 sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv;
762 }
763 } else {
764 for (i = 0; i < 16; i++)
765 sregs->u.s.ppc32.sr[i] = vcpu->arch.shared->sr[i];
766
767 for (i = 0; i < 8; i++) {
768 sregs->u.s.ppc32.ibat[i] = vcpu3s->ibat[i].raw;
769 sregs->u.s.ppc32.dbat[i] = vcpu3s->dbat[i].raw;
770 }
771 }
772
773 return 0;
774}
775
776int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
777 struct kvm_sregs *sregs)
778{
779 struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu);
780 int i;
781
782 kvmppc_set_pvr(vcpu, sregs->pvr);
783
784 vcpu3s->sdr1 = sregs->u.s.sdr1;
785 if (vcpu->arch.hflags & BOOK3S_HFLAG_SLB) {
786 for (i = 0; i < 64; i++) {
787 vcpu->arch.mmu.slbmte(vcpu, sregs->u.s.ppc64.slb[i].slbv,
788 sregs->u.s.ppc64.slb[i].slbe);
789 }
790 } else {
791 for (i = 0; i < 16; i++) {
792 vcpu->arch.mmu.mtsrin(vcpu, i, sregs->u.s.ppc32.sr[i]);
793 }
794 for (i = 0; i < 8; i++) {
795 kvmppc_set_bat(vcpu, &(vcpu3s->ibat[i]), false,
796 (u32)sregs->u.s.ppc32.ibat[i]);
797 kvmppc_set_bat(vcpu, &(vcpu3s->ibat[i]), true,
798 (u32)(sregs->u.s.ppc32.ibat[i] >> 32));
799 kvmppc_set_bat(vcpu, &(vcpu3s->dbat[i]), false,
800 (u32)sregs->u.s.ppc32.dbat[i]);
801 kvmppc_set_bat(vcpu, &(vcpu3s->dbat[i]), true,
802 (u32)(sregs->u.s.ppc32.dbat[i] >> 32));
803 }
804 }
805
806 /* Flush the MMU after messing with the segments */
807 kvmppc_mmu_pte_flush(vcpu, 0, 0);
808
809 return 0;
810}
811
812int kvmppc_core_check_processor_compat(void)
813{
814 return 0;
815}
816
817struct kvm_vcpu *kvmppc_core_vcpu_create(struct kvm *kvm, unsigned int id)
818{
819 struct kvmppc_vcpu_book3s *vcpu_book3s;
820 struct kvm_vcpu *vcpu;
821 int err = -ENOMEM;
822 unsigned long p;
823
824 vcpu_book3s = vzalloc(sizeof(struct kvmppc_vcpu_book3s));
825 if (!vcpu_book3s)
826 goto out;
827
828 vcpu_book3s->shadow_vcpu = (struct kvmppc_book3s_shadow_vcpu *)
829 kzalloc(sizeof(*vcpu_book3s->shadow_vcpu), GFP_KERNEL);
830 if (!vcpu_book3s->shadow_vcpu)
831 goto free_vcpu;
832
833 vcpu = &vcpu_book3s->vcpu;
834 err = kvm_vcpu_init(vcpu, kvm, id);
835 if (err)
836 goto free_shadow_vcpu;
837
838 p = __get_free_page(GFP_KERNEL|__GFP_ZERO);
839 /* the real shared page fills the last 4k of our page */
840 vcpu->arch.shared = (void*)(p + PAGE_SIZE - 4096);
841 if (!p)
842 goto uninit_vcpu;
843
844 vcpu->arch.host_retip = kvm_return_point;
845 vcpu->arch.host_msr = mfmsr();
846#ifdef CONFIG_PPC_BOOK3S_64
847 /* default to book3s_64 (970fx) */
848 vcpu->arch.pvr = 0x3C0301;
849#else
850 /* default to book3s_32 (750) */
851 vcpu->arch.pvr = 0x84202;
852#endif
853 kvmppc_set_pvr(vcpu, vcpu->arch.pvr);
854 vcpu->arch.slb_nr = 64;
855
856 /* remember where some real-mode handlers are */
857 vcpu->arch.trampoline_lowmem = __pa(kvmppc_handler_lowmem_trampoline);
858 vcpu->arch.trampoline_enter = __pa(kvmppc_handler_trampoline_enter);
859 vcpu->arch.highmem_handler = (ulong)kvmppc_handler_highmem;
860#ifdef CONFIG_PPC_BOOK3S_64
861 vcpu->arch.rmcall = *(ulong*)kvmppc_rmcall;
862#else
863 vcpu->arch.rmcall = (ulong)kvmppc_rmcall;
864#endif
865
866 vcpu->arch.shadow_msr = MSR_USER64;
867
868 err = kvmppc_mmu_init(vcpu);
869 if (err < 0)
870 goto uninit_vcpu;
871
872 return vcpu;
873
874uninit_vcpu:
875 kvm_vcpu_uninit(vcpu);
876free_shadow_vcpu:
877 kfree(vcpu_book3s->shadow_vcpu);
878free_vcpu:
879 vfree(vcpu_book3s);
880out:
881 return ERR_PTR(err);
882}
883
884void kvmppc_core_vcpu_free(struct kvm_vcpu *vcpu)
885{
886 struct kvmppc_vcpu_book3s *vcpu_book3s = to_book3s(vcpu);
887
888 free_page((unsigned long)vcpu->arch.shared & PAGE_MASK);
889 kvm_vcpu_uninit(vcpu);
890 kfree(vcpu_book3s->shadow_vcpu);
891 vfree(vcpu_book3s);
892}
893
894extern int __kvmppc_vcpu_entry(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu);
895int __kvmppc_vcpu_run(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
896{
897 int ret;
898 double fpr[32][TS_FPRWIDTH];
899 unsigned int fpscr;
900 int fpexc_mode;
901#ifdef CONFIG_ALTIVEC
902 vector128 vr[32];
903 vector128 vscr;
904 unsigned long uninitialized_var(vrsave);
905 int used_vr;
906#endif
907#ifdef CONFIG_VSX
908 int used_vsr;
909#endif
910 ulong ext_msr;
911
912 /* No need to go into the guest when all we do is going out */
913 if (signal_pending(current)) {
914 kvm_run->exit_reason = KVM_EXIT_INTR;
915 return -EINTR;
916 }
917
918 /* Save FPU state in stack */
919 if (current->thread.regs->msr & MSR_FP)
920 giveup_fpu(current);
921 memcpy(fpr, current->thread.fpr, sizeof(current->thread.fpr));
922 fpscr = current->thread.fpscr.val;
923 fpexc_mode = current->thread.fpexc_mode;
924
925#ifdef CONFIG_ALTIVEC
926 /* Save Altivec state in stack */
927 used_vr = current->thread.used_vr;
928 if (used_vr) {
929 if (current->thread.regs->msr & MSR_VEC)
930 giveup_altivec(current);
931 memcpy(vr, current->thread.vr, sizeof(current->thread.vr));
932 vscr = current->thread.vscr;
933 vrsave = current->thread.vrsave;
934 }
935#endif
936
937#ifdef CONFIG_VSX
938 /* Save VSX state in stack */
939 used_vsr = current->thread.used_vsr;
940 if (used_vsr && (current->thread.regs->msr & MSR_VSX))
941 __giveup_vsx(current);
942#endif
943
944 /* Remember the MSR with disabled extensions */
945 ext_msr = current->thread.regs->msr;
946
947 /* XXX we get called with irq disabled - change that! */
948 local_irq_enable();
949
950 /* Preload FPU if it's enabled */
951 if (vcpu->arch.shared->msr & MSR_FP)
952 kvmppc_handle_ext(vcpu, BOOK3S_INTERRUPT_FP_UNAVAIL, MSR_FP);
953
954 ret = __kvmppc_vcpu_entry(kvm_run, vcpu);
955
956 local_irq_disable();
957
958 current->thread.regs->msr = ext_msr;
959
960 /* Make sure we save the guest FPU/Altivec/VSX state */
961 kvmppc_giveup_ext(vcpu, MSR_FP);
962 kvmppc_giveup_ext(vcpu, MSR_VEC);
963 kvmppc_giveup_ext(vcpu, MSR_VSX);
964
965 /* Restore FPU state from stack */
966 memcpy(current->thread.fpr, fpr, sizeof(current->thread.fpr));
967 current->thread.fpscr.val = fpscr;
968 current->thread.fpexc_mode = fpexc_mode;
969
970#ifdef CONFIG_ALTIVEC
971 /* Restore Altivec state from stack */
972 if (used_vr && current->thread.used_vr) {
973 memcpy(current->thread.vr, vr, sizeof(current->thread.vr));
974 current->thread.vscr = vscr;
975 current->thread.vrsave = vrsave;
976 }
977 current->thread.used_vr = used_vr;
978#endif
979
980#ifdef CONFIG_VSX
981 current->thread.used_vsr = used_vsr;
982#endif
983
984 return ret;
985}
986
987static int kvmppc_book3s_init(void)
988{
989 int r;
990
991 r = kvm_init(NULL, sizeof(struct kvmppc_vcpu_book3s), 0,
992 THIS_MODULE);
993
994 if (r)
995 return r;
996
997 r = kvmppc_mmu_hpte_sysinit();
998
999 return r;
1000}
1001
1002static void kvmppc_book3s_exit(void)
1003{
1004 kvmppc_mmu_hpte_sysexit();
1005 kvm_exit();
1006}
1007
1008module_init(kvmppc_book3s_init);
1009module_exit(kvmppc_book3s_exit);
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-24 20:25:02 -0500