aboutsummaryrefslogtreecommitdiffstats
path: root/arch/powerpc
diff options
context:
space:
mode:
Diffstat (limited to 'arch/powerpc')
-rw-r--r--arch/powerpc/include/asm/cputable.h14
-rw-r--r--arch/powerpc/include/asm/exception-64s.h136
-rw-r--r--arch/powerpc/include/asm/hvcall.h5
-rw-r--r--arch/powerpc/include/asm/kvm.h15
-rw-r--r--arch/powerpc/include/asm/kvm_asm.h4
-rw-r--r--arch/powerpc/include/asm/kvm_book3s.h196
-rw-r--r--arch/powerpc/include/asm/kvm_book3s_64.h4
-rw-r--r--arch/powerpc/include/asm/kvm_book3s_asm.h41
-rw-r--r--arch/powerpc/include/asm/kvm_booke.h4
-rw-r--r--arch/powerpc/include/asm/kvm_e500.h30
-rw-r--r--arch/powerpc/include/asm/kvm_host.h169
-rw-r--r--arch/powerpc/include/asm/kvm_ppc.h41
-rw-r--r--arch/powerpc/include/asm/mmu-hash64.h10
-rw-r--r--arch/powerpc/include/asm/paca.h3
-rw-r--r--arch/powerpc/include/asm/ppc_asm.h28
-rw-r--r--arch/powerpc/include/asm/reg.h25
-rw-r--r--arch/powerpc/include/asm/reg_booke.h1
-rw-r--r--arch/powerpc/kernel/asm-offsets.c190
-rw-r--r--arch/powerpc/kernel/cpu_setup_power7.S22
-rw-r--r--arch/powerpc/kernel/cpu_setup_ppc970.S26
-rw-r--r--arch/powerpc/kernel/exceptions-64s.S228
-rw-r--r--arch/powerpc/kernel/head_fsl_booke.S8
-rw-r--r--arch/powerpc/kernel/idle_power7.S2
-rw-r--r--arch/powerpc/kernel/paca.c2
-rw-r--r--arch/powerpc/kernel/process.c4
-rw-r--r--arch/powerpc/kernel/setup-common.c3
-rw-r--r--arch/powerpc/kernel/setup_64.c3
-rw-r--r--arch/powerpc/kernel/smp.c1
-rw-r--r--arch/powerpc/kernel/traps.c5
-rw-r--r--arch/powerpc/kvm/44x_tlb.c4
-rw-r--r--arch/powerpc/kvm/Kconfig34
-rw-r--r--arch/powerpc/kvm/Makefile27
-rw-r--r--arch/powerpc/kvm/book3s.c1007
-rw-r--r--arch/powerpc/kvm/book3s_64_mmu.c54
-rw-r--r--arch/powerpc/kvm/book3s_64_mmu_hv.c180
-rw-r--r--arch/powerpc/kvm/book3s_64_vio_hv.c73
-rw-r--r--arch/powerpc/kvm/book3s_exports.c9
-rw-r--r--arch/powerpc/kvm/book3s_hv.c1269
-rw-r--r--arch/powerpc/kvm/book3s_hv_builtin.c155
-rw-r--r--arch/powerpc/kvm/book3s_hv_interrupts.S166
-rw-r--r--arch/powerpc/kvm/book3s_hv_rm_mmu.c370
-rw-r--r--arch/powerpc/kvm/book3s_hv_rmhandlers.S1345
-rw-r--r--arch/powerpc/kvm/book3s_interrupts.S21
-rw-r--r--arch/powerpc/kvm/book3s_mmu_hpte.c71
-rw-r--r--arch/powerpc/kvm/book3s_pr.c1029
-rw-r--r--arch/powerpc/kvm/book3s_rmhandlers.S102
-rw-r--r--arch/powerpc/kvm/book3s_segment.S117
-rw-r--r--arch/powerpc/kvm/booke.c132
-rw-r--r--arch/powerpc/kvm/booke.h23
-rw-r--r--arch/powerpc/kvm/booke_interrupts.S66
-rw-r--r--arch/powerpc/kvm/e500.c7
-rw-r--r--arch/powerpc/kvm/e500_emulate.c4
-rw-r--r--arch/powerpc/kvm/e500_tlb.c800
-rw-r--r--arch/powerpc/kvm/e500_tlb.h13
-rw-r--r--arch/powerpc/kvm/powerpc.c78
-rw-r--r--arch/powerpc/kvm/timing.c9
-rw-r--r--arch/powerpc/kvm/trace.h4
-rw-r--r--arch/powerpc/mm/hash_native_64.c6
-rw-r--r--arch/powerpc/platforms/iseries/exception.S2
-rw-r--r--arch/powerpc/platforms/iseries/exception.h4
-rw-r--r--arch/powerpc/sysdev/xics/icp-native.c9
61 files changed, 6631 insertions, 1779 deletions
diff --git a/arch/powerpc/include/asm/cputable.h b/arch/powerpc/include/asm/cputable.h
index c0d842cfd012..e30442c539ce 100644
--- a/arch/powerpc/include/asm/cputable.h
+++ b/arch/powerpc/include/asm/cputable.h
@@ -179,8 +179,9 @@ extern const char *powerpc_base_platform;
179#define LONG_ASM_CONST(x) 0 179#define LONG_ASM_CONST(x) 0
180#endif 180#endif
181 181
182 182#define CPU_FTR_HVMODE LONG_ASM_CONST(0x0000000200000000)
183#define CPU_FTR_HVMODE_206 LONG_ASM_CONST(0x0000000800000000) 183#define CPU_FTR_ARCH_201 LONG_ASM_CONST(0x0000000400000000)
184#define CPU_FTR_ARCH_206 LONG_ASM_CONST(0x0000000800000000)
184#define CPU_FTR_CFAR LONG_ASM_CONST(0x0000001000000000) 185#define CPU_FTR_CFAR LONG_ASM_CONST(0x0000001000000000)
185#define CPU_FTR_IABR LONG_ASM_CONST(0x0000002000000000) 186#define CPU_FTR_IABR LONG_ASM_CONST(0x0000002000000000)
186#define CPU_FTR_MMCRA LONG_ASM_CONST(0x0000004000000000) 187#define CPU_FTR_MMCRA LONG_ASM_CONST(0x0000004000000000)
@@ -401,9 +402,10 @@ extern const char *powerpc_base_platform;
401 CPU_FTR_MMCRA | CPU_FTR_CP_USE_DCBTZ | \ 402 CPU_FTR_MMCRA | CPU_FTR_CP_USE_DCBTZ | \
402 CPU_FTR_STCX_CHECKS_ADDRESS) 403 CPU_FTR_STCX_CHECKS_ADDRESS)
403#define CPU_FTRS_PPC970 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ 404#define CPU_FTRS_PPC970 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \
404 CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \ 405 CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | CPU_FTR_ARCH_201 | \
405 CPU_FTR_ALTIVEC_COMP | CPU_FTR_CAN_NAP | CPU_FTR_MMCRA | \ 406 CPU_FTR_ALTIVEC_COMP | CPU_FTR_CAN_NAP | CPU_FTR_MMCRA | \
406 CPU_FTR_CP_USE_DCBTZ | CPU_FTR_STCX_CHECKS_ADDRESS) 407 CPU_FTR_CP_USE_DCBTZ | CPU_FTR_STCX_CHECKS_ADDRESS | \
408 CPU_FTR_HVMODE)
407#define CPU_FTRS_POWER5 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ 409#define CPU_FTRS_POWER5 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \
408 CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \ 410 CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \
409 CPU_FTR_MMCRA | CPU_FTR_SMT | \ 411 CPU_FTR_MMCRA | CPU_FTR_SMT | \
@@ -417,13 +419,13 @@ extern const char *powerpc_base_platform;
417 CPU_FTR_DSCR | CPU_FTR_UNALIGNED_LD_STD | \ 419 CPU_FTR_DSCR | CPU_FTR_UNALIGNED_LD_STD | \
418 CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_CFAR) 420 CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_CFAR)
419#define CPU_FTRS_POWER7 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ 421#define CPU_FTRS_POWER7 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \
420 CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | CPU_FTR_HVMODE_206 |\ 422 CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | CPU_FTR_ARCH_206 |\
421 CPU_FTR_MMCRA | CPU_FTR_SMT | \ 423 CPU_FTR_MMCRA | CPU_FTR_SMT | \
422 CPU_FTR_COHERENT_ICACHE | \ 424 CPU_FTR_COHERENT_ICACHE | \
423 CPU_FTR_PURR | CPU_FTR_SPURR | CPU_FTR_REAL_LE | \ 425 CPU_FTR_PURR | CPU_FTR_SPURR | CPU_FTR_REAL_LE | \
424 CPU_FTR_DSCR | CPU_FTR_SAO | CPU_FTR_ASYM_SMT | \ 426 CPU_FTR_DSCR | CPU_FTR_SAO | CPU_FTR_ASYM_SMT | \
425 CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \ 427 CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \
426 CPU_FTR_ICSWX | CPU_FTR_CFAR) 428 CPU_FTR_ICSWX | CPU_FTR_CFAR | CPU_FTR_HVMODE)
427#define CPU_FTRS_CELL (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ 429#define CPU_FTRS_CELL (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \
428 CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \ 430 CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \
429 CPU_FTR_ALTIVEC_COMP | CPU_FTR_MMCRA | CPU_FTR_SMT | \ 431 CPU_FTR_ALTIVEC_COMP | CPU_FTR_MMCRA | CPU_FTR_SMT | \
diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h
index f5dfe3411f64..8057f4f6980f 100644
--- a/arch/powerpc/include/asm/exception-64s.h
+++ b/arch/powerpc/include/asm/exception-64s.h
@@ -61,19 +61,22 @@
61#define EXC_HV H 61#define EXC_HV H
62#define EXC_STD 62#define EXC_STD
63 63
64#define EXCEPTION_PROLOG_1(area) \ 64#define __EXCEPTION_PROLOG_1(area, extra, vec) \
65 GET_PACA(r13); \ 65 GET_PACA(r13); \
66 std r9,area+EX_R9(r13); /* save r9 - r12 */ \ 66 std r9,area+EX_R9(r13); /* save r9 - r12 */ \
67 std r10,area+EX_R10(r13); \ 67 std r10,area+EX_R10(r13); \
68 std r11,area+EX_R11(r13); \
69 std r12,area+EX_R12(r13); \
70 BEGIN_FTR_SECTION_NESTED(66); \ 68 BEGIN_FTR_SECTION_NESTED(66); \
71 mfspr r10,SPRN_CFAR; \ 69 mfspr r10,SPRN_CFAR; \
72 std r10,area+EX_CFAR(r13); \ 70 std r10,area+EX_CFAR(r13); \
73 END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 66); \ 71 END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 66); \
74 GET_SCRATCH0(r9); \ 72 mfcr r9; \
75 std r9,area+EX_R13(r13); \ 73 extra(vec); \
76 mfcr r9 74 std r11,area+EX_R11(r13); \
75 std r12,area+EX_R12(r13); \
76 GET_SCRATCH0(r10); \
77 std r10,area+EX_R13(r13)
78#define EXCEPTION_PROLOG_1(area, extra, vec) \
79 __EXCEPTION_PROLOG_1(area, extra, vec)
77 80
78#define __EXCEPTION_PROLOG_PSERIES_1(label, h) \ 81#define __EXCEPTION_PROLOG_PSERIES_1(label, h) \
79 ld r12,PACAKBASE(r13); /* get high part of &label */ \ 82 ld r12,PACAKBASE(r13); /* get high part of &label */ \
@@ -85,13 +88,65 @@
85 mtspr SPRN_##h##SRR1,r10; \ 88 mtspr SPRN_##h##SRR1,r10; \
86 h##rfid; \ 89 h##rfid; \
87 b . /* prevent speculative execution */ 90 b . /* prevent speculative execution */
88#define EXCEPTION_PROLOG_PSERIES_1(label, h) \ 91#define EXCEPTION_PROLOG_PSERIES_1(label, h) \
89 __EXCEPTION_PROLOG_PSERIES_1(label, h) 92 __EXCEPTION_PROLOG_PSERIES_1(label, h)
90 93
91#define EXCEPTION_PROLOG_PSERIES(area, label, h) \ 94#define EXCEPTION_PROLOG_PSERIES(area, label, h, extra, vec) \
92 EXCEPTION_PROLOG_1(area); \ 95 EXCEPTION_PROLOG_1(area, extra, vec); \
93 EXCEPTION_PROLOG_PSERIES_1(label, h); 96 EXCEPTION_PROLOG_PSERIES_1(label, h);
94 97
98#define __KVMTEST(n) \
99 lbz r10,HSTATE_IN_GUEST(r13); \
100 cmpwi r10,0; \
101 bne do_kvm_##n
102
103#define __KVM_HANDLER(area, h, n) \
104do_kvm_##n: \
105 ld r10,area+EX_R10(r13); \
106 stw r9,HSTATE_SCRATCH1(r13); \
107 ld r9,area+EX_R9(r13); \
108 std r12,HSTATE_SCRATCH0(r13); \
109 li r12,n; \
110 b kvmppc_interrupt
111
112#define __KVM_HANDLER_SKIP(area, h, n) \
113do_kvm_##n: \
114 cmpwi r10,KVM_GUEST_MODE_SKIP; \
115 ld r10,area+EX_R10(r13); \
116 beq 89f; \
117 stw r9,HSTATE_SCRATCH1(r13); \
118 ld r9,area+EX_R9(r13); \
119 std r12,HSTATE_SCRATCH0(r13); \
120 li r12,n; \
121 b kvmppc_interrupt; \
12289: mtocrf 0x80,r9; \
123 ld r9,area+EX_R9(r13); \
124 b kvmppc_skip_##h##interrupt
125
126#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
127#define KVMTEST(n) __KVMTEST(n)
128#define KVM_HANDLER(area, h, n) __KVM_HANDLER(area, h, n)
129#define KVM_HANDLER_SKIP(area, h, n) __KVM_HANDLER_SKIP(area, h, n)
130
131#else
132#define KVMTEST(n)
133#define KVM_HANDLER(area, h, n)
134#define KVM_HANDLER_SKIP(area, h, n)
135#endif
136
137#ifdef CONFIG_KVM_BOOK3S_PR
138#define KVMTEST_PR(n) __KVMTEST(n)
139#define KVM_HANDLER_PR(area, h, n) __KVM_HANDLER(area, h, n)
140#define KVM_HANDLER_PR_SKIP(area, h, n) __KVM_HANDLER_SKIP(area, h, n)
141
142#else
143#define KVMTEST_PR(n)
144#define KVM_HANDLER_PR(area, h, n)
145#define KVM_HANDLER_PR_SKIP(area, h, n)
146#endif
147
148#define NOTEST(n)
149
95/* 150/*
96 * The common exception prolog is used for all except a few exceptions 151 * The common exception prolog is used for all except a few exceptions
97 * such as a segment miss on a kernel address. We have to be prepared 152 * such as a segment miss on a kernel address. We have to be prepared
@@ -164,57 +219,58 @@
164 .globl label##_pSeries; \ 219 .globl label##_pSeries; \
165label##_pSeries: \ 220label##_pSeries: \
166 HMT_MEDIUM; \ 221 HMT_MEDIUM; \
167 DO_KVM vec; \
168 SET_SCRATCH0(r13); /* save r13 */ \ 222 SET_SCRATCH0(r13); /* save r13 */ \
169 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common, EXC_STD) 223 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common, \
224 EXC_STD, KVMTEST_PR, vec)
170 225
171#define STD_EXCEPTION_HV(loc, vec, label) \ 226#define STD_EXCEPTION_HV(loc, vec, label) \
172 . = loc; \ 227 . = loc; \
173 .globl label##_hv; \ 228 .globl label##_hv; \
174label##_hv: \ 229label##_hv: \
175 HMT_MEDIUM; \ 230 HMT_MEDIUM; \
176 DO_KVM vec; \ 231 SET_SCRATCH0(r13); /* save r13 */ \
177 SET_SCRATCH0(r13); /* save r13 */ \ 232 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common, \
178 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common, EXC_HV) 233 EXC_HV, KVMTEST, vec)
179 234
180#define __MASKABLE_EXCEPTION_PSERIES(vec, label, h) \ 235#define __SOFTEN_TEST(h) \
181 HMT_MEDIUM; \
182 DO_KVM vec; \
183 SET_SCRATCH0(r13); /* save r13 */ \
184 GET_PACA(r13); \
185 std r9,PACA_EXGEN+EX_R9(r13); /* save r9, r10 */ \
186 std r10,PACA_EXGEN+EX_R10(r13); \
187 lbz r10,PACASOFTIRQEN(r13); \ 236 lbz r10,PACASOFTIRQEN(r13); \
188 mfcr r9; \
189 cmpwi r10,0; \ 237 cmpwi r10,0; \
190 beq masked_##h##interrupt; \ 238 beq masked_##h##interrupt
191 GET_SCRATCH0(r10); \ 239#define _SOFTEN_TEST(h) __SOFTEN_TEST(h)
192 std r10,PACA_EXGEN+EX_R13(r13); \ 240
193 std r11,PACA_EXGEN+EX_R11(r13); \ 241#define SOFTEN_TEST_PR(vec) \
194 std r12,PACA_EXGEN+EX_R12(r13); \ 242 KVMTEST_PR(vec); \
195 ld r12,PACAKBASE(r13); /* get high part of &label */ \ 243 _SOFTEN_TEST(EXC_STD)
196 ld r10,PACAKMSR(r13); /* get MSR value for kernel */ \ 244
197 mfspr r11,SPRN_##h##SRR0; /* save SRR0 */ \ 245#define SOFTEN_TEST_HV(vec) \
198 LOAD_HANDLER(r12,label##_common) \ 246 KVMTEST(vec); \
199 mtspr SPRN_##h##SRR0,r12; \ 247 _SOFTEN_TEST(EXC_HV)
200 mfspr r12,SPRN_##h##SRR1; /* and SRR1 */ \ 248
201 mtspr SPRN_##h##SRR1,r10; \ 249#define SOFTEN_TEST_HV_201(vec) \
202 h##rfid; \ 250 KVMTEST(vec); \
203 b . /* prevent speculative execution */ 251 _SOFTEN_TEST(EXC_STD)
204#define _MASKABLE_EXCEPTION_PSERIES(vec, label, h) \ 252
205 __MASKABLE_EXCEPTION_PSERIES(vec, label, h) 253#define __MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra) \
254 HMT_MEDIUM; \
255 SET_SCRATCH0(r13); /* save r13 */ \
256 __EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec); \
257 EXCEPTION_PROLOG_PSERIES_1(label##_common, h);
258#define _MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra) \
259 __MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra)
206 260
207#define MASKABLE_EXCEPTION_PSERIES(loc, vec, label) \ 261#define MASKABLE_EXCEPTION_PSERIES(loc, vec, label) \
208 . = loc; \ 262 . = loc; \
209 .globl label##_pSeries; \ 263 .globl label##_pSeries; \
210label##_pSeries: \ 264label##_pSeries: \
211 _MASKABLE_EXCEPTION_PSERIES(vec, label, EXC_STD) 265 _MASKABLE_EXCEPTION_PSERIES(vec, label, \
266 EXC_STD, SOFTEN_TEST_PR)
212 267
213#define MASKABLE_EXCEPTION_HV(loc, vec, label) \ 268#define MASKABLE_EXCEPTION_HV(loc, vec, label) \
214 . = loc; \ 269 . = loc; \
215 .globl label##_hv; \ 270 .globl label##_hv; \
216label##_hv: \ 271label##_hv: \
217 _MASKABLE_EXCEPTION_PSERIES(vec, label, EXC_HV) 272 _MASKABLE_EXCEPTION_PSERIES(vec, label, \
273 EXC_HV, SOFTEN_TEST_HV)
218 274
219#ifdef CONFIG_PPC_ISERIES 275#ifdef CONFIG_PPC_ISERIES
220#define DISABLE_INTS \ 276#define DISABLE_INTS \
diff --git a/arch/powerpc/include/asm/hvcall.h b/arch/powerpc/include/asm/hvcall.h
index fd8201dddd4b..1c324ff55ea8 100644
--- a/arch/powerpc/include/asm/hvcall.h
+++ b/arch/powerpc/include/asm/hvcall.h
@@ -29,6 +29,10 @@
29#define H_LONG_BUSY_ORDER_100_SEC 9905 /* Long busy, hint that 100sec \ 29#define H_LONG_BUSY_ORDER_100_SEC 9905 /* Long busy, hint that 100sec \
30 is a good time to retry */ 30 is a good time to retry */
31#define H_LONG_BUSY_END_RANGE 9905 /* End of long busy range */ 31#define H_LONG_BUSY_END_RANGE 9905 /* End of long busy range */
32
33/* Internal value used in book3s_hv kvm support; not returned to guests */
34#define H_TOO_HARD 9999
35
32#define H_HARDWARE -1 /* Hardware error */ 36#define H_HARDWARE -1 /* Hardware error */
33#define H_FUNCTION -2 /* Function not supported */ 37#define H_FUNCTION -2 /* Function not supported */
34#define H_PRIVILEGE -3 /* Caller not privileged */ 38#define H_PRIVILEGE -3 /* Caller not privileged */
@@ -100,6 +104,7 @@
100#define H_PAGE_SET_ACTIVE H_PAGE_STATE_CHANGE 104#define H_PAGE_SET_ACTIVE H_PAGE_STATE_CHANGE
101#define H_AVPN (1UL<<(63-32)) /* An avpn is provided as a sanity test */ 105#define H_AVPN (1UL<<(63-32)) /* An avpn is provided as a sanity test */
102#define H_ANDCOND (1UL<<(63-33)) 106#define H_ANDCOND (1UL<<(63-33))
107#define H_LOCAL (1UL<<(63-35))
103#define H_ICACHE_INVALIDATE (1UL<<(63-40)) /* icbi, etc. (ignored for IO pages) */ 108#define H_ICACHE_INVALIDATE (1UL<<(63-40)) /* icbi, etc. (ignored for IO pages) */
104#define H_ICACHE_SYNCHRONIZE (1UL<<(63-41)) /* dcbst, icbi, etc (ignored for IO pages */ 109#define H_ICACHE_SYNCHRONIZE (1UL<<(63-41)) /* dcbst, icbi, etc (ignored for IO pages */
105#define H_COALESCE_CAND (1UL<<(63-42)) /* page is a good candidate for coalescing */ 110#define H_COALESCE_CAND (1UL<<(63-42)) /* page is a good candidate for coalescing */
diff --git a/arch/powerpc/include/asm/kvm.h b/arch/powerpc/include/asm/kvm.h
index d2ca5ed3877b..a4f6c85431f8 100644
--- a/arch/powerpc/include/asm/kvm.h
+++ b/arch/powerpc/include/asm/kvm.h
@@ -22,6 +22,10 @@
22 22
23#include <linux/types.h> 23#include <linux/types.h>
24 24
25/* Select powerpc specific features in <linux/kvm.h> */
26#define __KVM_HAVE_SPAPR_TCE
27#define __KVM_HAVE_PPC_SMT
28
25struct kvm_regs { 29struct kvm_regs {
26 __u64 pc; 30 __u64 pc;
27 __u64 cr; 31 __u64 cr;
@@ -272,4 +276,15 @@ struct kvm_guest_debug_arch {
272#define KVM_INTERRUPT_UNSET -2U 276#define KVM_INTERRUPT_UNSET -2U
273#define KVM_INTERRUPT_SET_LEVEL -3U 277#define KVM_INTERRUPT_SET_LEVEL -3U
274 278
279/* for KVM_CAP_SPAPR_TCE */
280struct kvm_create_spapr_tce {
281 __u64 liobn;
282 __u32 window_size;
283};
284
285/* for KVM_ALLOCATE_RMA */
286struct kvm_allocate_rma {
287 __u64 rma_size;
288};
289
275#endif /* __LINUX_KVM_POWERPC_H */ 290#endif /* __LINUX_KVM_POWERPC_H */
diff --git a/arch/powerpc/include/asm/kvm_asm.h b/arch/powerpc/include/asm/kvm_asm.h
index 0951b17f4eb5..7b1f0e0fc653 100644
--- a/arch/powerpc/include/asm/kvm_asm.h
+++ b/arch/powerpc/include/asm/kvm_asm.h
@@ -64,8 +64,12 @@
64#define BOOK3S_INTERRUPT_PROGRAM 0x700 64#define BOOK3S_INTERRUPT_PROGRAM 0x700
65#define BOOK3S_INTERRUPT_FP_UNAVAIL 0x800 65#define BOOK3S_INTERRUPT_FP_UNAVAIL 0x800
66#define BOOK3S_INTERRUPT_DECREMENTER 0x900 66#define BOOK3S_INTERRUPT_DECREMENTER 0x900
67#define BOOK3S_INTERRUPT_HV_DECREMENTER 0x980
67#define BOOK3S_INTERRUPT_SYSCALL 0xc00 68#define BOOK3S_INTERRUPT_SYSCALL 0xc00
68#define BOOK3S_INTERRUPT_TRACE 0xd00 69#define BOOK3S_INTERRUPT_TRACE 0xd00
70#define BOOK3S_INTERRUPT_H_DATA_STORAGE 0xe00
71#define BOOK3S_INTERRUPT_H_INST_STORAGE 0xe20
72#define BOOK3S_INTERRUPT_H_EMUL_ASSIST 0xe40
69#define BOOK3S_INTERRUPT_PERFMON 0xf00 73#define BOOK3S_INTERRUPT_PERFMON 0xf00
70#define BOOK3S_INTERRUPT_ALTIVEC 0xf20 74#define BOOK3S_INTERRUPT_ALTIVEC 0xf20
71#define BOOK3S_INTERRUPT_VSX 0xf40 75#define BOOK3S_INTERRUPT_VSX 0xf40
diff --git a/arch/powerpc/include/asm/kvm_book3s.h b/arch/powerpc/include/asm/kvm_book3s.h
index d62e703f1214..98da010252a3 100644
--- a/arch/powerpc/include/asm/kvm_book3s.h
+++ b/arch/powerpc/include/asm/kvm_book3s.h
@@ -24,20 +24,6 @@
24#include <linux/kvm_host.h> 24#include <linux/kvm_host.h>
25#include <asm/kvm_book3s_asm.h> 25#include <asm/kvm_book3s_asm.h>
26 26
27struct kvmppc_slb {
28 u64 esid;
29 u64 vsid;
30 u64 orige;
31 u64 origv;
32 bool valid : 1;
33 bool Ks : 1;
34 bool Kp : 1;
35 bool nx : 1;
36 bool large : 1; /* PTEs are 16MB */
37 bool tb : 1; /* 1TB segment */
38 bool class : 1;
39};
40
41struct kvmppc_bat { 27struct kvmppc_bat {
42 u64 raw; 28 u64 raw;
43 u32 bepi; 29 u32 bepi;
@@ -67,11 +53,22 @@ struct kvmppc_sid_map {
67#define VSID_POOL_SIZE (SID_CONTEXTS * 16) 53#define VSID_POOL_SIZE (SID_CONTEXTS * 16)
68#endif 54#endif
69 55
56struct hpte_cache {
57 struct hlist_node list_pte;
58 struct hlist_node list_pte_long;
59 struct hlist_node list_vpte;
60 struct hlist_node list_vpte_long;
61 struct rcu_head rcu_head;
62 u64 host_va;
63 u64 pfn;
64 ulong slot;
65 struct kvmppc_pte pte;
66};
67
70struct kvmppc_vcpu_book3s { 68struct kvmppc_vcpu_book3s {
71 struct kvm_vcpu vcpu; 69 struct kvm_vcpu vcpu;
72 struct kvmppc_book3s_shadow_vcpu *shadow_vcpu; 70 struct kvmppc_book3s_shadow_vcpu *shadow_vcpu;
73 struct kvmppc_sid_map sid_map[SID_MAP_NUM]; 71 struct kvmppc_sid_map sid_map[SID_MAP_NUM];
74 struct kvmppc_slb slb[64];
75 struct { 72 struct {
76 u64 esid; 73 u64 esid;
77 u64 vsid; 74 u64 vsid;
@@ -81,7 +78,6 @@ struct kvmppc_vcpu_book3s {
81 struct kvmppc_bat dbat[8]; 78 struct kvmppc_bat dbat[8];
82 u64 hid[6]; 79 u64 hid[6];
83 u64 gqr[8]; 80 u64 gqr[8];
84 int slb_nr;
85 u64 sdr1; 81 u64 sdr1;
86 u64 hior; 82 u64 hior;
87 u64 msr_mask; 83 u64 msr_mask;
@@ -93,7 +89,13 @@ struct kvmppc_vcpu_book3s {
93 u64 vsid_max; 89 u64 vsid_max;
94#endif 90#endif
95 int context_id[SID_CONTEXTS]; 91 int context_id[SID_CONTEXTS];
96 ulong prog_flags; /* flags to inject when giving a 700 trap */ 92
93 struct hlist_head hpte_hash_pte[HPTEG_HASH_NUM_PTE];
94 struct hlist_head hpte_hash_pte_long[HPTEG_HASH_NUM_PTE_LONG];
95 struct hlist_head hpte_hash_vpte[HPTEG_HASH_NUM_VPTE];
96 struct hlist_head hpte_hash_vpte_long[HPTEG_HASH_NUM_VPTE_LONG];
97 int hpte_cache_count;
98 spinlock_t mmu_lock;
97}; 99};
98 100
99#define CONTEXT_HOST 0 101#define CONTEXT_HOST 0
@@ -110,8 +112,10 @@ extern void kvmppc_mmu_pte_flush(struct kvm_vcpu *vcpu, ulong ea, ulong ea_mask)
110extern void kvmppc_mmu_pte_vflush(struct kvm_vcpu *vcpu, u64 vp, u64 vp_mask); 112extern void kvmppc_mmu_pte_vflush(struct kvm_vcpu *vcpu, u64 vp, u64 vp_mask);
111extern void kvmppc_mmu_pte_pflush(struct kvm_vcpu *vcpu, ulong pa_start, ulong pa_end); 113extern void kvmppc_mmu_pte_pflush(struct kvm_vcpu *vcpu, ulong pa_start, ulong pa_end);
112extern void kvmppc_set_msr(struct kvm_vcpu *vcpu, u64 new_msr); 114extern void kvmppc_set_msr(struct kvm_vcpu *vcpu, u64 new_msr);
115extern void kvmppc_set_pvr(struct kvm_vcpu *vcpu, u32 pvr);
113extern void kvmppc_mmu_book3s_64_init(struct kvm_vcpu *vcpu); 116extern void kvmppc_mmu_book3s_64_init(struct kvm_vcpu *vcpu);
114extern void kvmppc_mmu_book3s_32_init(struct kvm_vcpu *vcpu); 117extern void kvmppc_mmu_book3s_32_init(struct kvm_vcpu *vcpu);
118extern void kvmppc_mmu_book3s_hv_init(struct kvm_vcpu *vcpu);
115extern int kvmppc_mmu_map_page(struct kvm_vcpu *vcpu, struct kvmppc_pte *pte); 119extern int kvmppc_mmu_map_page(struct kvm_vcpu *vcpu, struct kvmppc_pte *pte);
116extern int kvmppc_mmu_map_segment(struct kvm_vcpu *vcpu, ulong eaddr); 120extern int kvmppc_mmu_map_segment(struct kvm_vcpu *vcpu, ulong eaddr);
117extern void kvmppc_mmu_flush_segments(struct kvm_vcpu *vcpu); 121extern void kvmppc_mmu_flush_segments(struct kvm_vcpu *vcpu);
@@ -123,19 +127,22 @@ extern int kvmppc_mmu_hpte_init(struct kvm_vcpu *vcpu);
123extern void kvmppc_mmu_invalidate_pte(struct kvm_vcpu *vcpu, struct hpte_cache *pte); 127extern void kvmppc_mmu_invalidate_pte(struct kvm_vcpu *vcpu, struct hpte_cache *pte);
124extern int kvmppc_mmu_hpte_sysinit(void); 128extern int kvmppc_mmu_hpte_sysinit(void);
125extern void kvmppc_mmu_hpte_sysexit(void); 129extern void kvmppc_mmu_hpte_sysexit(void);
130extern int kvmppc_mmu_hv_init(void);
126 131
127extern int kvmppc_ld(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr, bool data); 132extern int kvmppc_ld(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr, bool data);
128extern int kvmppc_st(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr, bool data); 133extern int kvmppc_st(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr, bool data);
129extern void kvmppc_book3s_queue_irqprio(struct kvm_vcpu *vcpu, unsigned int vec); 134extern void kvmppc_book3s_queue_irqprio(struct kvm_vcpu *vcpu, unsigned int vec);
135extern void kvmppc_inject_interrupt(struct kvm_vcpu *vcpu, int vec, u64 flags);
130extern void kvmppc_set_bat(struct kvm_vcpu *vcpu, struct kvmppc_bat *bat, 136extern void kvmppc_set_bat(struct kvm_vcpu *vcpu, struct kvmppc_bat *bat,
131 bool upper, u32 val); 137 bool upper, u32 val);
132extern void kvmppc_giveup_ext(struct kvm_vcpu *vcpu, ulong msr); 138extern void kvmppc_giveup_ext(struct kvm_vcpu *vcpu, ulong msr);
133extern int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu); 139extern int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu);
134extern pfn_t kvmppc_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn); 140extern pfn_t kvmppc_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn);
135 141
136extern ulong kvmppc_trampoline_lowmem; 142extern void kvmppc_handler_lowmem_trampoline(void);
137extern ulong kvmppc_trampoline_enter; 143extern void kvmppc_handler_trampoline_enter(void);
138extern void kvmppc_rmcall(ulong srr0, ulong srr1); 144extern void kvmppc_rmcall(ulong srr0, ulong srr1);
145extern void kvmppc_hv_entry_trampoline(void);
139extern void kvmppc_load_up_fpu(void); 146extern void kvmppc_load_up_fpu(void);
140extern void kvmppc_load_up_altivec(void); 147extern void kvmppc_load_up_altivec(void);
141extern void kvmppc_load_up_vsx(void); 148extern void kvmppc_load_up_vsx(void);
@@ -147,15 +154,32 @@ static inline struct kvmppc_vcpu_book3s *to_book3s(struct kvm_vcpu *vcpu)
147 return container_of(vcpu, struct kvmppc_vcpu_book3s, vcpu); 154 return container_of(vcpu, struct kvmppc_vcpu_book3s, vcpu);
148} 155}
149 156
150static inline ulong dsisr(void) 157extern void kvm_return_point(void);
158
159/* Also add subarch specific defines */
160
161#ifdef CONFIG_KVM_BOOK3S_32_HANDLER
162#include <asm/kvm_book3s_32.h>
163#endif
164#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
165#include <asm/kvm_book3s_64.h>
166#endif
167
168#ifdef CONFIG_KVM_BOOK3S_PR
169
170static inline unsigned long kvmppc_interrupt_offset(struct kvm_vcpu *vcpu)
151{ 171{
152 ulong r; 172 return to_book3s(vcpu)->hior;
153 asm ( "mfdsisr %0 " : "=r" (r) );
154 return r;
155} 173}
156 174
157extern void kvm_return_point(void); 175static inline void kvmppc_update_int_pending(struct kvm_vcpu *vcpu,
158static inline struct kvmppc_book3s_shadow_vcpu *to_svcpu(struct kvm_vcpu *vcpu); 176 unsigned long pending_now, unsigned long old_pending)
177{
178 if (pending_now)
179 vcpu->arch.shared->int_pending = 1;
180 else if (old_pending)
181 vcpu->arch.shared->int_pending = 0;
182}
159 183
160static inline void kvmppc_set_gpr(struct kvm_vcpu *vcpu, int num, ulong val) 184static inline void kvmppc_set_gpr(struct kvm_vcpu *vcpu, int num, ulong val)
161{ 185{
@@ -244,6 +268,120 @@ static inline ulong kvmppc_get_fault_dar(struct kvm_vcpu *vcpu)
244 return to_svcpu(vcpu)->fault_dar; 268 return to_svcpu(vcpu)->fault_dar;
245} 269}
246 270
271static inline bool kvmppc_critical_section(struct kvm_vcpu *vcpu)
272{
273 ulong crit_raw = vcpu->arch.shared->critical;
274 ulong crit_r1 = kvmppc_get_gpr(vcpu, 1);
275 bool crit;
276
277 /* Truncate crit indicators in 32 bit mode */
278 if (!(vcpu->arch.shared->msr & MSR_SF)) {
279 crit_raw &= 0xffffffff;
280 crit_r1 &= 0xffffffff;
281 }
282
283 /* Critical section when crit == r1 */
284 crit = (crit_raw == crit_r1);
285 /* ... and we're in supervisor mode */
286 crit = crit && !(vcpu->arch.shared->msr & MSR_PR);
287
288 return crit;
289}
290#else /* CONFIG_KVM_BOOK3S_PR */
291
292static inline unsigned long kvmppc_interrupt_offset(struct kvm_vcpu *vcpu)
293{
294 return 0;
295}
296
297static inline void kvmppc_update_int_pending(struct kvm_vcpu *vcpu,
298 unsigned long pending_now, unsigned long old_pending)
299{
300}
301
302static inline void kvmppc_set_gpr(struct kvm_vcpu *vcpu, int num, ulong val)
303{
304 vcpu->arch.gpr[num] = val;
305}
306
307static inline ulong kvmppc_get_gpr(struct kvm_vcpu *vcpu, int num)
308{
309 return vcpu->arch.gpr[num];
310}
311
312static inline void kvmppc_set_cr(struct kvm_vcpu *vcpu, u32 val)
313{
314 vcpu->arch.cr = val;
315}
316
317static inline u32 kvmppc_get_cr(struct kvm_vcpu *vcpu)
318{
319 return vcpu->arch.cr;
320}
321
322static inline void kvmppc_set_xer(struct kvm_vcpu *vcpu, u32 val)
323{
324 vcpu->arch.xer = val;
325}
326
327static inline u32 kvmppc_get_xer(struct kvm_vcpu *vcpu)
328{
329 return vcpu->arch.xer;
330}
331
332static inline void kvmppc_set_ctr(struct kvm_vcpu *vcpu, ulong val)
333{
334 vcpu->arch.ctr = val;
335}
336
337static inline ulong kvmppc_get_ctr(struct kvm_vcpu *vcpu)
338{
339 return vcpu->arch.ctr;
340}
341
342static inline void kvmppc_set_lr(struct kvm_vcpu *vcpu, ulong val)
343{
344 vcpu->arch.lr = val;
345}
346
347static inline ulong kvmppc_get_lr(struct kvm_vcpu *vcpu)
348{
349 return vcpu->arch.lr;
350}
351
352static inline void kvmppc_set_pc(struct kvm_vcpu *vcpu, ulong val)
353{
354 vcpu->arch.pc = val;
355}
356
357static inline ulong kvmppc_get_pc(struct kvm_vcpu *vcpu)
358{
359 return vcpu->arch.pc;
360}
361
362static inline u32 kvmppc_get_last_inst(struct kvm_vcpu *vcpu)
363{
364 ulong pc = kvmppc_get_pc(vcpu);
365
366 /* Load the instruction manually if it failed to do so in the
367 * exit path */
368 if (vcpu->arch.last_inst == KVM_INST_FETCH_FAILED)
369 kvmppc_ld(vcpu, &pc, sizeof(u32), &vcpu->arch.last_inst, false);
370
371 return vcpu->arch.last_inst;
372}
373
374static inline ulong kvmppc_get_fault_dar(struct kvm_vcpu *vcpu)
375{
376 return vcpu->arch.fault_dar;
377}
378
379static inline bool kvmppc_critical_section(struct kvm_vcpu *vcpu)
380{
381 return false;
382}
383#endif
384
247/* Magic register values loaded into r3 and r4 before the 'sc' assembly 385/* Magic register values loaded into r3 and r4 before the 'sc' assembly
248 * instruction for the OSI hypercalls */ 386 * instruction for the OSI hypercalls */
249#define OSI_SC_MAGIC_R3 0x113724FA 387#define OSI_SC_MAGIC_R3 0x113724FA
@@ -251,12 +389,4 @@ static inline ulong kvmppc_get_fault_dar(struct kvm_vcpu *vcpu)
251 389
252#define INS_DCBZ 0x7c0007ec 390#define INS_DCBZ 0x7c0007ec
253 391
254/* Also add subarch specific defines */
255
256#ifdef CONFIG_PPC_BOOK3S_32
257#include <asm/kvm_book3s_32.h>
258#else
259#include <asm/kvm_book3s_64.h>
260#endif
261
262#endif /* __ASM_KVM_BOOK3S_H__ */ 392#endif /* __ASM_KVM_BOOK3S_H__ */
diff --git a/arch/powerpc/include/asm/kvm_book3s_64.h b/arch/powerpc/include/asm/kvm_book3s_64.h
index 4cadd612d575..e43fe42b9875 100644
--- a/arch/powerpc/include/asm/kvm_book3s_64.h
+++ b/arch/powerpc/include/asm/kvm_book3s_64.h
@@ -20,9 +20,13 @@
20#ifndef __ASM_KVM_BOOK3S_64_H__ 20#ifndef __ASM_KVM_BOOK3S_64_H__
21#define __ASM_KVM_BOOK3S_64_H__ 21#define __ASM_KVM_BOOK3S_64_H__
22 22
23#ifdef CONFIG_KVM_BOOK3S_PR
23static inline struct kvmppc_book3s_shadow_vcpu *to_svcpu(struct kvm_vcpu *vcpu) 24static inline struct kvmppc_book3s_shadow_vcpu *to_svcpu(struct kvm_vcpu *vcpu)
24{ 25{
25 return &get_paca()->shadow_vcpu; 26 return &get_paca()->shadow_vcpu;
26} 27}
28#endif
29
30#define SPAPR_TCE_SHIFT 12
27 31
28#endif /* __ASM_KVM_BOOK3S_64_H__ */ 32#endif /* __ASM_KVM_BOOK3S_64_H__ */
diff --git a/arch/powerpc/include/asm/kvm_book3s_asm.h b/arch/powerpc/include/asm/kvm_book3s_asm.h
index d5a8a3861635..ef7b3688c3b6 100644
--- a/arch/powerpc/include/asm/kvm_book3s_asm.h
+++ b/arch/powerpc/include/asm/kvm_book3s_asm.h
@@ -60,6 +60,36 @@ kvmppc_resume_\intno:
60 60
61#else /*__ASSEMBLY__ */ 61#else /*__ASSEMBLY__ */
62 62
63/*
64 * This struct goes in the PACA on 64-bit processors. It is used
65 * to store host state that needs to be saved when we enter a guest
66 * and restored when we exit, but isn't specific to any particular
67 * guest or vcpu. It also has some scratch fields used by the guest
68 * exit code.
69 */
70struct kvmppc_host_state {
71 ulong host_r1;
72 ulong host_r2;
73 ulong host_msr;
74 ulong vmhandler;
75 ulong scratch0;
76 ulong scratch1;
77 u8 in_guest;
78
79#ifdef CONFIG_KVM_BOOK3S_64_HV
80 struct kvm_vcpu *kvm_vcpu;
81 struct kvmppc_vcore *kvm_vcore;
82 unsigned long xics_phys;
83 u64 dabr;
84 u64 host_mmcr[3];
85 u32 host_pmc[8];
86 u64 host_purr;
87 u64 host_spurr;
88 u64 host_dscr;
89 u64 dec_expires;
90#endif
91};
92
63struct kvmppc_book3s_shadow_vcpu { 93struct kvmppc_book3s_shadow_vcpu {
64 ulong gpr[14]; 94 ulong gpr[14];
65 u32 cr; 95 u32 cr;
@@ -73,17 +103,12 @@ struct kvmppc_book3s_shadow_vcpu {
73 ulong shadow_srr1; 103 ulong shadow_srr1;
74 ulong fault_dar; 104 ulong fault_dar;
75 105
76 ulong host_r1;
77 ulong host_r2;
78 ulong handler;
79 ulong scratch0;
80 ulong scratch1;
81 ulong vmhandler;
82 u8 in_guest;
83
84#ifdef CONFIG_PPC_BOOK3S_32 106#ifdef CONFIG_PPC_BOOK3S_32
85 u32 sr[16]; /* Guest SRs */ 107 u32 sr[16]; /* Guest SRs */
108
109 struct kvmppc_host_state hstate;
86#endif 110#endif
111
87#ifdef CONFIG_PPC_BOOK3S_64 112#ifdef CONFIG_PPC_BOOK3S_64
88 u8 slb_max; /* highest used guest slb entry */ 113 u8 slb_max; /* highest used guest slb entry */
89 struct { 114 struct {
diff --git a/arch/powerpc/include/asm/kvm_booke.h b/arch/powerpc/include/asm/kvm_booke.h
index 9c9ba3d59b1b..a90e09188777 100644
--- a/arch/powerpc/include/asm/kvm_booke.h
+++ b/arch/powerpc/include/asm/kvm_booke.h
@@ -93,4 +93,8 @@ static inline ulong kvmppc_get_fault_dar(struct kvm_vcpu *vcpu)
93 return vcpu->arch.fault_dear; 93 return vcpu->arch.fault_dear;
94} 94}
95 95
96static inline ulong kvmppc_get_msr(struct kvm_vcpu *vcpu)
97{
98 return vcpu->arch.shared->msr;
99}
96#endif /* __ASM_KVM_BOOKE_H__ */ 100#endif /* __ASM_KVM_BOOKE_H__ */
diff --git a/arch/powerpc/include/asm/kvm_e500.h b/arch/powerpc/include/asm/kvm_e500.h
index 7a2a565f88c4..adbfca9dd100 100644
--- a/arch/powerpc/include/asm/kvm_e500.h
+++ b/arch/powerpc/include/asm/kvm_e500.h
@@ -1,5 +1,5 @@
1/* 1/*
2 * Copyright (C) 2008 Freescale Semiconductor, Inc. All rights reserved. 2 * Copyright (C) 2008-2011 Freescale Semiconductor, Inc. All rights reserved.
3 * 3 *
4 * Author: Yu Liu, <yu.liu@freescale.com> 4 * Author: Yu Liu, <yu.liu@freescale.com>
5 * 5 *
@@ -29,17 +29,25 @@ struct tlbe{
29 u32 mas7; 29 u32 mas7;
30}; 30};
31 31
32#define E500_TLB_VALID 1
33#define E500_TLB_DIRTY 2
34
35struct tlbe_priv {
36 pfn_t pfn;
37 unsigned int flags; /* E500_TLB_* */
38};
39
40struct vcpu_id_table;
41
32struct kvmppc_vcpu_e500 { 42struct kvmppc_vcpu_e500 {
33 /* Unmodified copy of the guest's TLB. */ 43 /* Unmodified copy of the guest's TLB. */
34 struct tlbe *guest_tlb[E500_TLB_NUM]; 44 struct tlbe *gtlb_arch[E500_TLB_NUM];
35 /* TLB that's actually used when the guest is running. */
36 struct tlbe *shadow_tlb[E500_TLB_NUM];
37 /* Pages which are referenced in the shadow TLB. */
38 struct page **shadow_pages[E500_TLB_NUM];
39 45
40 unsigned int guest_tlb_size[E500_TLB_NUM]; 46 /* KVM internal information associated with each guest TLB entry */
41 unsigned int shadow_tlb_size[E500_TLB_NUM]; 47 struct tlbe_priv *gtlb_priv[E500_TLB_NUM];
42 unsigned int guest_tlb_nv[E500_TLB_NUM]; 48
49 unsigned int gtlb_size[E500_TLB_NUM];
50 unsigned int gtlb_nv[E500_TLB_NUM];
43 51
44 u32 host_pid[E500_PID_NUM]; 52 u32 host_pid[E500_PID_NUM];
45 u32 pid[E500_PID_NUM]; 53 u32 pid[E500_PID_NUM];
@@ -53,6 +61,10 @@ struct kvmppc_vcpu_e500 {
53 u32 mas5; 61 u32 mas5;
54 u32 mas6; 62 u32 mas6;
55 u32 mas7; 63 u32 mas7;
64
65 /* vcpu id table */
66 struct vcpu_id_table *idt;
67
56 u32 l1csr0; 68 u32 l1csr0;
57 u32 l1csr1; 69 u32 l1csr1;
58 u32 hid0; 70 u32 hid0;
diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h
index 186f150b9b89..cc22b282d755 100644
--- a/arch/powerpc/include/asm/kvm_host.h
+++ b/arch/powerpc/include/asm/kvm_host.h
@@ -25,15 +25,23 @@
25#include <linux/interrupt.h> 25#include <linux/interrupt.h>
26#include <linux/types.h> 26#include <linux/types.h>
27#include <linux/kvm_types.h> 27#include <linux/kvm_types.h>
28#include <linux/threads.h>
29#include <linux/spinlock.h>
28#include <linux/kvm_para.h> 30#include <linux/kvm_para.h>
31#include <linux/list.h>
32#include <linux/atomic.h>
29#include <asm/kvm_asm.h> 33#include <asm/kvm_asm.h>
34#include <asm/processor.h>
30 35
31#define KVM_MAX_VCPUS 1 36#define KVM_MAX_VCPUS NR_CPUS
37#define KVM_MAX_VCORES NR_CPUS
32#define KVM_MEMORY_SLOTS 32 38#define KVM_MEMORY_SLOTS 32
33/* memory slots that does not exposed to userspace */ 39/* memory slots that does not exposed to userspace */
34#define KVM_PRIVATE_MEM_SLOTS 4 40#define KVM_PRIVATE_MEM_SLOTS 4
35 41
42#ifdef CONFIG_KVM_MMIO
36#define KVM_COALESCED_MMIO_PAGE_OFFSET 1 43#define KVM_COALESCED_MMIO_PAGE_OFFSET 1
44#endif
37 45
38/* We don't currently support large pages. */ 46/* We don't currently support large pages. */
39#define KVM_HPAGE_GFN_SHIFT(x) 0 47#define KVM_HPAGE_GFN_SHIFT(x) 0
@@ -57,6 +65,10 @@ struct kvm;
57struct kvm_run; 65struct kvm_run;
58struct kvm_vcpu; 66struct kvm_vcpu;
59 67
68struct lppaca;
69struct slb_shadow;
70struct dtl;
71
60struct kvm_vm_stat { 72struct kvm_vm_stat {
61 u32 remote_tlb_flush; 73 u32 remote_tlb_flush;
62}; 74};
@@ -133,9 +145,74 @@ struct kvmppc_exit_timing {
133 }; 145 };
134}; 146};
135 147
148struct kvmppc_pginfo {
149 unsigned long pfn;
150 atomic_t refcnt;
151};
152
153struct kvmppc_spapr_tce_table {
154 struct list_head list;
155 struct kvm *kvm;
156 u64 liobn;
157 u32 window_size;
158 struct page *pages[0];
159};
160
161struct kvmppc_rma_info {
162 void *base_virt;
163 unsigned long base_pfn;
164 unsigned long npages;
165 struct list_head list;
166 atomic_t use_count;
167};
168
136struct kvm_arch { 169struct kvm_arch {
170#ifdef CONFIG_KVM_BOOK3S_64_HV
171 unsigned long hpt_virt;
172 unsigned long ram_npages;
173 unsigned long ram_psize;
174 unsigned long ram_porder;
175 struct kvmppc_pginfo *ram_pginfo;
176 unsigned int lpid;
177 unsigned int host_lpid;
178 unsigned long host_lpcr;
179 unsigned long sdr1;
180 unsigned long host_sdr1;
181 int tlbie_lock;
182 int n_rma_pages;
183 unsigned long lpcr;
184 unsigned long rmor;
185 struct kvmppc_rma_info *rma;
186 struct list_head spapr_tce_tables;
187 unsigned short last_vcpu[NR_CPUS];
188 struct kvmppc_vcore *vcores[KVM_MAX_VCORES];
189#endif /* CONFIG_KVM_BOOK3S_64_HV */
137}; 190};
138 191
192/*
193 * Struct for a virtual core.
194 * Note: entry_exit_count combines an entry count in the bottom 8 bits
195 * and an exit count in the next 8 bits. This is so that we can
196 * atomically increment the entry count iff the exit count is 0
197 * without taking the lock.
198 */
199struct kvmppc_vcore {
200 int n_runnable;
201 int n_blocked;
202 int num_threads;
203 int entry_exit_count;
204 int n_woken;
205 int nap_count;
206 u16 pcpu;
207 u8 vcore_running;
208 u8 in_guest;
209 struct list_head runnable_threads;
210 spinlock_t lock;
211};
212
213#define VCORE_ENTRY_COUNT(vc) ((vc)->entry_exit_count & 0xff)
214#define VCORE_EXIT_COUNT(vc) ((vc)->entry_exit_count >> 8)
215
139struct kvmppc_pte { 216struct kvmppc_pte {
140 ulong eaddr; 217 ulong eaddr;
141 u64 vpage; 218 u64 vpage;
@@ -163,16 +240,18 @@ struct kvmppc_mmu {
163 bool (*is_dcbz32)(struct kvm_vcpu *vcpu); 240 bool (*is_dcbz32)(struct kvm_vcpu *vcpu);
164}; 241};
165 242
166struct hpte_cache { 243struct kvmppc_slb {
167 struct hlist_node list_pte; 244 u64 esid;
168 struct hlist_node list_pte_long; 245 u64 vsid;
169 struct hlist_node list_vpte; 246 u64 orige;
170 struct hlist_node list_vpte_long; 247 u64 origv;
171 struct rcu_head rcu_head; 248 bool valid : 1;
172 u64 host_va; 249 bool Ks : 1;
173 u64 pfn; 250 bool Kp : 1;
174 ulong slot; 251 bool nx : 1;
175 struct kvmppc_pte pte; 252 bool large : 1; /* PTEs are 16MB */
253 bool tb : 1; /* 1TB segment */
254 bool class : 1;
176}; 255};
177 256
178struct kvm_vcpu_arch { 257struct kvm_vcpu_arch {
@@ -187,6 +266,9 @@ struct kvm_vcpu_arch {
187 ulong highmem_handler; 266 ulong highmem_handler;
188 ulong rmcall; 267 ulong rmcall;
189 ulong host_paca_phys; 268 ulong host_paca_phys;
269 struct kvmppc_slb slb[64];
270 int slb_max; /* 1 + index of last valid entry in slb[] */
271 int slb_nr; /* total number of entries in SLB */
190 struct kvmppc_mmu mmu; 272 struct kvmppc_mmu mmu;
191#endif 273#endif
192 274
@@ -195,13 +277,19 @@ struct kvm_vcpu_arch {
195 u64 fpr[32]; 277 u64 fpr[32];
196 u64 fpscr; 278 u64 fpscr;
197 279
280#ifdef CONFIG_SPE
281 ulong evr[32];
282 ulong spefscr;
283 ulong host_spefscr;
284 u64 acc;
285#endif
198#ifdef CONFIG_ALTIVEC 286#ifdef CONFIG_ALTIVEC
199 vector128 vr[32]; 287 vector128 vr[32];
200 vector128 vscr; 288 vector128 vscr;
201#endif 289#endif
202 290
203#ifdef CONFIG_VSX 291#ifdef CONFIG_VSX
204 u64 vsr[32]; 292 u64 vsr[64];
205#endif 293#endif
206 294
207#ifdef CONFIG_PPC_BOOK3S 295#ifdef CONFIG_PPC_BOOK3S
@@ -209,22 +297,27 @@ struct kvm_vcpu_arch {
209 u32 qpr[32]; 297 u32 qpr[32];
210#endif 298#endif
211 299
212#ifdef CONFIG_BOOKE
213 ulong pc; 300 ulong pc;
214 ulong ctr; 301 ulong ctr;
215 ulong lr; 302 ulong lr;
216 303
217 ulong xer; 304 ulong xer;
218 u32 cr; 305 u32 cr;
219#endif
220 306
221#ifdef CONFIG_PPC_BOOK3S 307#ifdef CONFIG_PPC_BOOK3S
222 ulong shadow_msr;
223 ulong hflags; 308 ulong hflags;
224 ulong guest_owned_ext; 309 ulong guest_owned_ext;
310 ulong purr;
311 ulong spurr;
312 ulong dscr;
313 ulong amr;
314 ulong uamor;
315 u32 ctrl;
316 ulong dabr;
225#endif 317#endif
226 u32 vrsave; /* also USPRG0 */ 318 u32 vrsave; /* also USPRG0 */
227 u32 mmucr; 319 u32 mmucr;
320 ulong shadow_msr;
228 ulong sprg4; 321 ulong sprg4;
229 ulong sprg5; 322 ulong sprg5;
230 ulong sprg6; 323 ulong sprg6;
@@ -249,6 +342,7 @@ struct kvm_vcpu_arch {
249 u32 pvr; 342 u32 pvr;
250 343
251 u32 shadow_pid; 344 u32 shadow_pid;
345 u32 shadow_pid1;
252 u32 pid; 346 u32 pid;
253 u32 swap_pid; 347 u32 swap_pid;
254 348
@@ -258,6 +352,9 @@ struct kvm_vcpu_arch {
258 u32 dbcr1; 352 u32 dbcr1;
259 u32 dbsr; 353 u32 dbsr;
260 354
355 u64 mmcr[3];
356 u32 pmc[8];
357
261#ifdef CONFIG_KVM_EXIT_TIMING 358#ifdef CONFIG_KVM_EXIT_TIMING
262 struct mutex exit_timing_lock; 359 struct mutex exit_timing_lock;
263 struct kvmppc_exit_timing timing_exit; 360 struct kvmppc_exit_timing timing_exit;
@@ -272,8 +369,12 @@ struct kvm_vcpu_arch {
272 struct dentry *debugfs_exit_timing; 369 struct dentry *debugfs_exit_timing;
273#endif 370#endif
274 371
372#ifdef CONFIG_PPC_BOOK3S
373 ulong fault_dar;
374 u32 fault_dsisr;
375#endif
376
275#ifdef CONFIG_BOOKE 377#ifdef CONFIG_BOOKE
276 u32 last_inst;
277 ulong fault_dear; 378 ulong fault_dear;
278 ulong fault_esr; 379 ulong fault_esr;
279 ulong queued_dear; 380 ulong queued_dear;
@@ -288,25 +389,47 @@ struct kvm_vcpu_arch {
288 u8 dcr_is_write; 389 u8 dcr_is_write;
289 u8 osi_needed; 390 u8 osi_needed;
290 u8 osi_enabled; 391 u8 osi_enabled;
392 u8 hcall_needed;
291 393
292 u32 cpr0_cfgaddr; /* holds the last set cpr0_cfgaddr */ 394 u32 cpr0_cfgaddr; /* holds the last set cpr0_cfgaddr */
293 395
294 struct hrtimer dec_timer; 396 struct hrtimer dec_timer;
295 struct tasklet_struct tasklet; 397 struct tasklet_struct tasklet;
296 u64 dec_jiffies; 398 u64 dec_jiffies;
399 u64 dec_expires;
297 unsigned long pending_exceptions; 400 unsigned long pending_exceptions;
401 u16 last_cpu;
402 u8 ceded;
403 u8 prodded;
404 u32 last_inst;
405
406 struct lppaca *vpa;
407 struct slb_shadow *slb_shadow;
408 struct dtl *dtl;
409 struct dtl *dtl_end;
410
411 struct kvmppc_vcore *vcore;
412 int ret;
413 int trap;
414 int state;
415 int ptid;
416 wait_queue_head_t cpu_run;
417
298 struct kvm_vcpu_arch_shared *shared; 418 struct kvm_vcpu_arch_shared *shared;
299 unsigned long magic_page_pa; /* phys addr to map the magic page to */ 419 unsigned long magic_page_pa; /* phys addr to map the magic page to */
300 unsigned long magic_page_ea; /* effect. addr to map the magic page to */ 420 unsigned long magic_page_ea; /* effect. addr to map the magic page to */
301 421
302#ifdef CONFIG_PPC_BOOK3S 422#ifdef CONFIG_KVM_BOOK3S_64_HV
303 struct hlist_head hpte_hash_pte[HPTEG_HASH_NUM_PTE]; 423 struct kvm_vcpu_arch_shared shregs;
304 struct hlist_head hpte_hash_pte_long[HPTEG_HASH_NUM_PTE_LONG]; 424
305 struct hlist_head hpte_hash_vpte[HPTEG_HASH_NUM_VPTE]; 425 struct list_head run_list;
306 struct hlist_head hpte_hash_vpte_long[HPTEG_HASH_NUM_VPTE_LONG]; 426 struct task_struct *run_task;
307 int hpte_cache_count; 427 struct kvm_run *kvm_run;
308 spinlock_t mmu_lock;
309#endif 428#endif
310}; 429};
311 430
431#define KVMPPC_VCPU_BUSY_IN_HOST 0
432#define KVMPPC_VCPU_BLOCKED 1
433#define KVMPPC_VCPU_RUNNABLE 2
434
312#endif /* __POWERPC_KVM_HOST_H__ */ 435#endif /* __POWERPC_KVM_HOST_H__ */
diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h
index 9345238edecf..d121f49d62b8 100644
--- a/arch/powerpc/include/asm/kvm_ppc.h
+++ b/arch/powerpc/include/asm/kvm_ppc.h
@@ -33,6 +33,9 @@
33#else 33#else
34#include <asm/kvm_booke.h> 34#include <asm/kvm_booke.h>
35#endif 35#endif
36#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
37#include <asm/paca.h>
38#endif
36 39
37enum emulation_result { 40enum emulation_result {
38 EMULATE_DONE, /* no further processing */ 41 EMULATE_DONE, /* no further processing */
@@ -42,6 +45,7 @@ enum emulation_result {
42 EMULATE_AGAIN, /* something went wrong. go again */ 45 EMULATE_AGAIN, /* something went wrong. go again */
43}; 46};
44 47
48extern int kvmppc_vcpu_run(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu);
45extern int __kvmppc_vcpu_run(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu); 49extern int __kvmppc_vcpu_run(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu);
46extern char kvmppc_handlers_start[]; 50extern char kvmppc_handlers_start[];
47extern unsigned long kvmppc_handler_len; 51extern unsigned long kvmppc_handler_len;
@@ -109,6 +113,27 @@ extern void kvmppc_booke_exit(void);
109 113
110extern void kvmppc_core_destroy_mmu(struct kvm_vcpu *vcpu); 114extern void kvmppc_core_destroy_mmu(struct kvm_vcpu *vcpu);
111extern int kvmppc_kvm_pv(struct kvm_vcpu *vcpu); 115extern int kvmppc_kvm_pv(struct kvm_vcpu *vcpu);
116extern void kvmppc_map_magic(struct kvm_vcpu *vcpu);
117
118extern long kvmppc_alloc_hpt(struct kvm *kvm);
119extern void kvmppc_free_hpt(struct kvm *kvm);
120extern long kvmppc_prepare_vrma(struct kvm *kvm,
121 struct kvm_userspace_memory_region *mem);
122extern void kvmppc_map_vrma(struct kvm *kvm,
123 struct kvm_userspace_memory_region *mem);
124extern int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu);
125extern long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
126 struct kvm_create_spapr_tce *args);
127extern long kvm_vm_ioctl_allocate_rma(struct kvm *kvm,
128 struct kvm_allocate_rma *rma);
129extern struct kvmppc_rma_info *kvm_alloc_rma(void);
130extern void kvm_release_rma(struct kvmppc_rma_info *ri);
131extern int kvmppc_core_init_vm(struct kvm *kvm);
132extern void kvmppc_core_destroy_vm(struct kvm *kvm);
133extern int kvmppc_core_prepare_memory_region(struct kvm *kvm,
134 struct kvm_userspace_memory_region *mem);
135extern void kvmppc_core_commit_memory_region(struct kvm *kvm,
136 struct kvm_userspace_memory_region *mem);
112 137
113/* 138/*
114 * Cuts out inst bits with ordering according to spec. 139 * Cuts out inst bits with ordering according to spec.
@@ -151,4 +176,20 @@ int kvmppc_set_sregs_ivor(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs);
151 176
152void kvmppc_set_pid(struct kvm_vcpu *vcpu, u32 pid); 177void kvmppc_set_pid(struct kvm_vcpu *vcpu, u32 pid);
153 178
179#ifdef CONFIG_KVM_BOOK3S_64_HV
180static inline void kvmppc_set_xics_phys(int cpu, unsigned long addr)
181{
182 paca[cpu].kvm_hstate.xics_phys = addr;
183}
184
185extern void kvm_rma_init(void);
186
187#else
188static inline void kvmppc_set_xics_phys(int cpu, unsigned long addr)
189{}
190
191static inline void kvm_rma_init(void)
192{}
193#endif
194
154#endif /* __POWERPC_KVM_PPC_H__ */ 195#endif /* __POWERPC_KVM_PPC_H__ */
diff --git a/arch/powerpc/include/asm/mmu-hash64.h b/arch/powerpc/include/asm/mmu-hash64.h
index d865bd909c7d..b445e0af4c2b 100644
--- a/arch/powerpc/include/asm/mmu-hash64.h
+++ b/arch/powerpc/include/asm/mmu-hash64.h
@@ -90,13 +90,19 @@ extern char initial_stab[];
90 90
91#define HPTE_R_PP0 ASM_CONST(0x8000000000000000) 91#define HPTE_R_PP0 ASM_CONST(0x8000000000000000)
92#define HPTE_R_TS ASM_CONST(0x4000000000000000) 92#define HPTE_R_TS ASM_CONST(0x4000000000000000)
93#define HPTE_R_KEY_HI ASM_CONST(0x3000000000000000)
93#define HPTE_R_RPN_SHIFT 12 94#define HPTE_R_RPN_SHIFT 12
94#define HPTE_R_RPN ASM_CONST(0x3ffffffffffff000) 95#define HPTE_R_RPN ASM_CONST(0x0ffffffffffff000)
95#define HPTE_R_FLAGS ASM_CONST(0x00000000000003ff)
96#define HPTE_R_PP ASM_CONST(0x0000000000000003) 96#define HPTE_R_PP ASM_CONST(0x0000000000000003)
97#define HPTE_R_N ASM_CONST(0x0000000000000004) 97#define HPTE_R_N ASM_CONST(0x0000000000000004)
98#define HPTE_R_G ASM_CONST(0x0000000000000008)
99#define HPTE_R_M ASM_CONST(0x0000000000000010)
100#define HPTE_R_I ASM_CONST(0x0000000000000020)
101#define HPTE_R_W ASM_CONST(0x0000000000000040)
102#define HPTE_R_WIMG ASM_CONST(0x0000000000000078)
98#define HPTE_R_C ASM_CONST(0x0000000000000080) 103#define HPTE_R_C ASM_CONST(0x0000000000000080)
99#define HPTE_R_R ASM_CONST(0x0000000000000100) 104#define HPTE_R_R ASM_CONST(0x0000000000000100)
105#define HPTE_R_KEY_LO ASM_CONST(0x0000000000000e00)
100 106
101#define HPTE_V_1TB_SEG ASM_CONST(0x4000000000000000) 107#define HPTE_V_1TB_SEG ASM_CONST(0x4000000000000000)
102#define HPTE_V_VRMA_MASK ASM_CONST(0x4001ffffff000000) 108#define HPTE_V_VRMA_MASK ASM_CONST(0x4001ffffff000000)
diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h
index 74126765106a..a6da12859959 100644
--- a/arch/powerpc/include/asm/paca.h
+++ b/arch/powerpc/include/asm/paca.h
@@ -147,9 +147,12 @@ struct paca_struct {
147 struct dtl_entry *dtl_curr; /* pointer corresponding to dtl_ridx */ 147 struct dtl_entry *dtl_curr; /* pointer corresponding to dtl_ridx */
148 148
149#ifdef CONFIG_KVM_BOOK3S_HANDLER 149#ifdef CONFIG_KVM_BOOK3S_HANDLER
150#ifdef CONFIG_KVM_BOOK3S_PR
150 /* We use this to store guest state in */ 151 /* We use this to store guest state in */
151 struct kvmppc_book3s_shadow_vcpu shadow_vcpu; 152 struct kvmppc_book3s_shadow_vcpu shadow_vcpu;
152#endif 153#endif
154 struct kvmppc_host_state kvm_hstate;
155#endif
153}; 156};
154 157
155extern struct paca_struct *paca; 158extern struct paca_struct *paca;
diff --git a/arch/powerpc/include/asm/ppc_asm.h b/arch/powerpc/include/asm/ppc_asm.h
index 1b422381fc16..368f72f79808 100644
--- a/arch/powerpc/include/asm/ppc_asm.h
+++ b/arch/powerpc/include/asm/ppc_asm.h
@@ -150,18 +150,22 @@ END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
150#define REST_16VSRSU(n,b,base) REST_8VSRSU(n,b,base); REST_8VSRSU(n+8,b,base) 150#define REST_16VSRSU(n,b,base) REST_8VSRSU(n,b,base); REST_8VSRSU(n+8,b,base)
151#define REST_32VSRSU(n,b,base) REST_16VSRSU(n,b,base); REST_16VSRSU(n+16,b,base) 151#define REST_32VSRSU(n,b,base) REST_16VSRSU(n,b,base); REST_16VSRSU(n+16,b,base)
152 152
153#define SAVE_EVR(n,s,base) evmergehi s,s,n; stw s,THREAD_EVR0+4*(n)(base) 153/*
154#define SAVE_2EVRS(n,s,base) SAVE_EVR(n,s,base); SAVE_EVR(n+1,s,base) 154 * b = base register for addressing, o = base offset from register of 1st EVR
155#define SAVE_4EVRS(n,s,base) SAVE_2EVRS(n,s,base); SAVE_2EVRS(n+2,s,base) 155 * n = first EVR, s = scratch
156#define SAVE_8EVRS(n,s,base) SAVE_4EVRS(n,s,base); SAVE_4EVRS(n+4,s,base) 156 */
157#define SAVE_16EVRS(n,s,base) SAVE_8EVRS(n,s,base); SAVE_8EVRS(n+8,s,base) 157#define SAVE_EVR(n,s,b,o) evmergehi s,s,n; stw s,o+4*(n)(b)
158#define SAVE_32EVRS(n,s,base) SAVE_16EVRS(n,s,base); SAVE_16EVRS(n+16,s,base) 158#define SAVE_2EVRS(n,s,b,o) SAVE_EVR(n,s,b,o); SAVE_EVR(n+1,s,b,o)
159#define REST_EVR(n,s,base) lwz s,THREAD_EVR0+4*(n)(base); evmergelo n,s,n 159#define SAVE_4EVRS(n,s,b,o) SAVE_2EVRS(n,s,b,o); SAVE_2EVRS(n+2,s,b,o)
160#define REST_2EVRS(n,s,base) REST_EVR(n,s,base); REST_EVR(n+1,s,base) 160#define SAVE_8EVRS(n,s,b,o) SAVE_4EVRS(n,s,b,o); SAVE_4EVRS(n+4,s,b,o)
161#define REST_4EVRS(n,s,base) REST_2EVRS(n,s,base); REST_2EVRS(n+2,s,base) 161#define SAVE_16EVRS(n,s,b,o) SAVE_8EVRS(n,s,b,o); SAVE_8EVRS(n+8,s,b,o)
162#define REST_8EVRS(n,s,base) REST_4EVRS(n,s,base); REST_4EVRS(n+4,s,base) 162#define SAVE_32EVRS(n,s,b,o) SAVE_16EVRS(n,s,b,o); SAVE_16EVRS(n+16,s,b,o)
163#define REST_16EVRS(n,s,base) REST_8EVRS(n,s,base); REST_8EVRS(n+8,s,base) 163#define REST_EVR(n,s,b,o) lwz s,o+4*(n)(b); evmergelo n,s,n
164#define REST_32EVRS(n,s,base) REST_16EVRS(n,s,base); REST_16EVRS(n+16,s,base) 164#define REST_2EVRS(n,s,b,o) REST_EVR(n,s,b,o); REST_EVR(n+1,s,b,o)
165#define REST_4EVRS(n,s,b,o) REST_2EVRS(n,s,b,o); REST_2EVRS(n+2,s,b,o)
166#define REST_8EVRS(n,s,b,o) REST_4EVRS(n,s,b,o); REST_4EVRS(n+4,s,b,o)
167#define REST_16EVRS(n,s,b,o) REST_8EVRS(n,s,b,o); REST_8EVRS(n+8,s,b,o)
168#define REST_32EVRS(n,s,b,o) REST_16EVRS(n,s,b,o); REST_16EVRS(n+16,s,b,o)
165 169
166/* Macros to adjust thread priority for hardware multithreading */ 170/* Macros to adjust thread priority for hardware multithreading */
167#define HMT_VERY_LOW or 31,31,31 # very low priority 171#define HMT_VERY_LOW or 31,31,31 # very low priority
diff --git a/arch/powerpc/include/asm/reg.h b/arch/powerpc/include/asm/reg.h
index c5cae0dd176c..ddbe57ae8584 100644
--- a/arch/powerpc/include/asm/reg.h
+++ b/arch/powerpc/include/asm/reg.h
@@ -189,6 +189,9 @@
189#define SPRN_CTR 0x009 /* Count Register */ 189#define SPRN_CTR 0x009 /* Count Register */
190#define SPRN_DSCR 0x11 190#define SPRN_DSCR 0x11
191#define SPRN_CFAR 0x1c /* Come From Address Register */ 191#define SPRN_CFAR 0x1c /* Come From Address Register */
192#define SPRN_AMR 0x1d /* Authority Mask Register */
193#define SPRN_UAMOR 0x9d /* User Authority Mask Override Register */
194#define SPRN_AMOR 0x15d /* Authority Mask Override Register */
192#define SPRN_ACOP 0x1F /* Available Coprocessor Register */ 195#define SPRN_ACOP 0x1F /* Available Coprocessor Register */
193#define SPRN_CTRLF 0x088 196#define SPRN_CTRLF 0x088
194#define SPRN_CTRLT 0x098 197#define SPRN_CTRLT 0x098
@@ -232,22 +235,28 @@
232#define LPCR_VPM0 (1ul << (63-0)) 235#define LPCR_VPM0 (1ul << (63-0))
233#define LPCR_VPM1 (1ul << (63-1)) 236#define LPCR_VPM1 (1ul << (63-1))
234#define LPCR_ISL (1ul << (63-2)) 237#define LPCR_ISL (1ul << (63-2))
238#define LPCR_VC_SH (63-2)
235#define LPCR_DPFD_SH (63-11) 239#define LPCR_DPFD_SH (63-11)
236#define LPCR_VRMA_L (1ul << (63-12)) 240#define LPCR_VRMA_L (1ul << (63-12))
237#define LPCR_VRMA_LP0 (1ul << (63-15)) 241#define LPCR_VRMA_LP0 (1ul << (63-15))
238#define LPCR_VRMA_LP1 (1ul << (63-16)) 242#define LPCR_VRMA_LP1 (1ul << (63-16))
243#define LPCR_VRMASD_SH (63-16)
239#define LPCR_RMLS 0x1C000000 /* impl dependent rmo limit sel */ 244#define LPCR_RMLS 0x1C000000 /* impl dependent rmo limit sel */
245#define LPCR_RMLS_SH (63-37)
240#define LPCR_ILE 0x02000000 /* !HV irqs set MSR:LE */ 246#define LPCR_ILE 0x02000000 /* !HV irqs set MSR:LE */
241#define LPCR_PECE 0x00007000 /* powersave exit cause enable */ 247#define LPCR_PECE 0x00007000 /* powersave exit cause enable */
242#define LPCR_PECE0 0x00004000 /* ext. exceptions can cause exit */ 248#define LPCR_PECE0 0x00004000 /* ext. exceptions can cause exit */
243#define LPCR_PECE1 0x00002000 /* decrementer can cause exit */ 249#define LPCR_PECE1 0x00002000 /* decrementer can cause exit */
244#define LPCR_PECE2 0x00001000 /* machine check etc can cause exit */ 250#define LPCR_PECE2 0x00001000 /* machine check etc can cause exit */
245#define LPCR_MER 0x00000800 /* Mediated External Exception */ 251#define LPCR_MER 0x00000800 /* Mediated External Exception */
252#define LPCR_LPES 0x0000000c
246#define LPCR_LPES0 0x00000008 /* LPAR Env selector 0 */ 253#define LPCR_LPES0 0x00000008 /* LPAR Env selector 0 */
247#define LPCR_LPES1 0x00000004 /* LPAR Env selector 1 */ 254#define LPCR_LPES1 0x00000004 /* LPAR Env selector 1 */
255#define LPCR_LPES_SH 2
248#define LPCR_RMI 0x00000002 /* real mode is cache inhibit */ 256#define LPCR_RMI 0x00000002 /* real mode is cache inhibit */
249#define LPCR_HDICE 0x00000001 /* Hyp Decr enable (HV,PR,EE) */ 257#define LPCR_HDICE 0x00000001 /* Hyp Decr enable (HV,PR,EE) */
250#define SPRN_LPID 0x13F /* Logical Partition Identifier */ 258#define SPRN_LPID 0x13F /* Logical Partition Identifier */
259#define LPID_RSVD 0x3ff /* Reserved LPID for partn switching */
251#define SPRN_HMER 0x150 /* Hardware m? error recovery */ 260#define SPRN_HMER 0x150 /* Hardware m? error recovery */
252#define SPRN_HMEER 0x151 /* Hardware m? enable error recovery */ 261#define SPRN_HMEER 0x151 /* Hardware m? enable error recovery */
253#define SPRN_HEIR 0x153 /* Hypervisor Emulated Instruction Register */ 262#define SPRN_HEIR 0x153 /* Hypervisor Emulated Instruction Register */
@@ -298,6 +307,7 @@
298#define SPRN_HASH1 0x3D2 /* Primary Hash Address Register */ 307#define SPRN_HASH1 0x3D2 /* Primary Hash Address Register */
299#define SPRN_HASH2 0x3D3 /* Secondary Hash Address Resgister */ 308#define SPRN_HASH2 0x3D3 /* Secondary Hash Address Resgister */
300#define SPRN_HID0 0x3F0 /* Hardware Implementation Register 0 */ 309#define SPRN_HID0 0x3F0 /* Hardware Implementation Register 0 */
310#define HID0_HDICE_SH (63 - 23) /* 970 HDEC interrupt enable */
301#define HID0_EMCP (1<<31) /* Enable Machine Check pin */ 311#define HID0_EMCP (1<<31) /* Enable Machine Check pin */
302#define HID0_EBA (1<<29) /* Enable Bus Address Parity */ 312#define HID0_EBA (1<<29) /* Enable Bus Address Parity */
303#define HID0_EBD (1<<28) /* Enable Bus Data Parity */ 313#define HID0_EBD (1<<28) /* Enable Bus Data Parity */
@@ -353,6 +363,13 @@
353#define SPRN_IABR2 0x3FA /* 83xx */ 363#define SPRN_IABR2 0x3FA /* 83xx */
354#define SPRN_IBCR 0x135 /* 83xx Insn Breakpoint Control Reg */ 364#define SPRN_IBCR 0x135 /* 83xx Insn Breakpoint Control Reg */
355#define SPRN_HID4 0x3F4 /* 970 HID4 */ 365#define SPRN_HID4 0x3F4 /* 970 HID4 */
366#define HID4_LPES0 (1ul << (63-0)) /* LPAR env. sel. bit 0 */
367#define HID4_RMLS2_SH (63 - 2) /* Real mode limit bottom 2 bits */
368#define HID4_LPID5_SH (63 - 6) /* partition ID bottom 4 bits */
369#define HID4_RMOR_SH (63 - 22) /* real mode offset (16 bits) */
370#define HID4_LPES1 (1 << (63-57)) /* LPAR env. sel. bit 1 */
371#define HID4_RMLS0_SH (63 - 58) /* Real mode limit top bit */
372#define HID4_LPID1_SH 0 /* partition ID top 2 bits */
356#define SPRN_HID4_GEKKO 0x3F3 /* Gekko HID4 */ 373#define SPRN_HID4_GEKKO 0x3F3 /* Gekko HID4 */
357#define SPRN_HID5 0x3F6 /* 970 HID5 */ 374#define SPRN_HID5 0x3F6 /* 970 HID5 */
358#define SPRN_HID6 0x3F9 /* BE HID 6 */ 375#define SPRN_HID6 0x3F9 /* BE HID 6 */
@@ -802,28 +819,28 @@
802 mfspr rX,SPRN_SPRG_PACA; \ 819 mfspr rX,SPRN_SPRG_PACA; \
803 FTR_SECTION_ELSE_NESTED(66); \ 820 FTR_SECTION_ELSE_NESTED(66); \
804 mfspr rX,SPRN_SPRG_HPACA; \ 821 mfspr rX,SPRN_SPRG_HPACA; \
805 ALT_FTR_SECTION_END_NESTED_IFCLR(CPU_FTR_HVMODE_206, 66) 822 ALT_FTR_SECTION_END_NESTED_IFCLR(CPU_FTR_HVMODE, 66)
806 823
807#define SET_PACA(rX) \ 824#define SET_PACA(rX) \
808 BEGIN_FTR_SECTION_NESTED(66); \ 825 BEGIN_FTR_SECTION_NESTED(66); \
809 mtspr SPRN_SPRG_PACA,rX; \ 826 mtspr SPRN_SPRG_PACA,rX; \
810 FTR_SECTION_ELSE_NESTED(66); \ 827 FTR_SECTION_ELSE_NESTED(66); \
811 mtspr SPRN_SPRG_HPACA,rX; \ 828 mtspr SPRN_SPRG_HPACA,rX; \
812 ALT_FTR_SECTION_END_NESTED_IFCLR(CPU_FTR_HVMODE_206, 66) 829 ALT_FTR_SECTION_END_NESTED_IFCLR(CPU_FTR_HVMODE, 66)
813 830
814#define GET_SCRATCH0(rX) \ 831#define GET_SCRATCH0(rX) \
815 BEGIN_FTR_SECTION_NESTED(66); \ 832 BEGIN_FTR_SECTION_NESTED(66); \
816 mfspr rX,SPRN_SPRG_SCRATCH0; \ 833 mfspr rX,SPRN_SPRG_SCRATCH0; \
817 FTR_SECTION_ELSE_NESTED(66); \ 834 FTR_SECTION_ELSE_NESTED(66); \
818 mfspr rX,SPRN_SPRG_HSCRATCH0; \ 835 mfspr rX,SPRN_SPRG_HSCRATCH0; \
819 ALT_FTR_SECTION_END_NESTED_IFCLR(CPU_FTR_HVMODE_206, 66) 836 ALT_FTR_SECTION_END_NESTED_IFCLR(CPU_FTR_HVMODE, 66)
820 837
821#define SET_SCRATCH0(rX) \ 838#define SET_SCRATCH0(rX) \
822 BEGIN_FTR_SECTION_NESTED(66); \ 839 BEGIN_FTR_SECTION_NESTED(66); \
823 mtspr SPRN_SPRG_SCRATCH0,rX; \ 840 mtspr SPRN_SPRG_SCRATCH0,rX; \
824 FTR_SECTION_ELSE_NESTED(66); \ 841 FTR_SECTION_ELSE_NESTED(66); \
825 mtspr SPRN_SPRG_HSCRATCH0,rX; \ 842 mtspr SPRN_SPRG_HSCRATCH0,rX; \
826 ALT_FTR_SECTION_END_NESTED_IFCLR(CPU_FTR_HVMODE_206, 66) 843 ALT_FTR_SECTION_END_NESTED_IFCLR(CPU_FTR_HVMODE, 66)
827 844
828#else /* CONFIG_PPC_BOOK3S_64 */ 845#else /* CONFIG_PPC_BOOK3S_64 */
829#define GET_SCRATCH0(rX) mfspr rX,SPRN_SPRG_SCRATCH0 846#define GET_SCRATCH0(rX) mfspr rX,SPRN_SPRG_SCRATCH0
diff --git a/arch/powerpc/include/asm/reg_booke.h b/arch/powerpc/include/asm/reg_booke.h
index 0f0ad9fa01c1..9ec0b39f9ddc 100644
--- a/arch/powerpc/include/asm/reg_booke.h
+++ b/arch/powerpc/include/asm/reg_booke.h
@@ -318,6 +318,7 @@
318#define ESR_ILK 0x00100000 /* Instr. Cache Locking */ 318#define ESR_ILK 0x00100000 /* Instr. Cache Locking */
319#define ESR_PUO 0x00040000 /* Unimplemented Operation exception */ 319#define ESR_PUO 0x00040000 /* Unimplemented Operation exception */
320#define ESR_BO 0x00020000 /* Byte Ordering */ 320#define ESR_BO 0x00020000 /* Byte Ordering */
321#define ESR_SPV 0x00000080 /* Signal Processing operation */
321 322
322/* Bit definitions related to the DBCR0. */ 323/* Bit definitions related to the DBCR0. */
323#if defined(CONFIG_40x) 324#if defined(CONFIG_40x)
diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index 36e1c8a29be8..54b935f2f5de 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -128,6 +128,7 @@ int main(void)
128 DEFINE(ICACHEL1LINESPERPAGE, offsetof(struct ppc64_caches, ilines_per_page)); 128 DEFINE(ICACHEL1LINESPERPAGE, offsetof(struct ppc64_caches, ilines_per_page));
129 /* paca */ 129 /* paca */
130 DEFINE(PACA_SIZE, sizeof(struct paca_struct)); 130 DEFINE(PACA_SIZE, sizeof(struct paca_struct));
131 DEFINE(PACA_LOCK_TOKEN, offsetof(struct paca_struct, lock_token));
131 DEFINE(PACAPACAINDEX, offsetof(struct paca_struct, paca_index)); 132 DEFINE(PACAPACAINDEX, offsetof(struct paca_struct, paca_index));
132 DEFINE(PACAPROCSTART, offsetof(struct paca_struct, cpu_start)); 133 DEFINE(PACAPROCSTART, offsetof(struct paca_struct, cpu_start));
133 DEFINE(PACAKSAVE, offsetof(struct paca_struct, kstack)); 134 DEFINE(PACAKSAVE, offsetof(struct paca_struct, kstack));
@@ -187,7 +188,9 @@ int main(void)
187 DEFINE(LPPACASRR1, offsetof(struct lppaca, saved_srr1)); 188 DEFINE(LPPACASRR1, offsetof(struct lppaca, saved_srr1));
188 DEFINE(LPPACAANYINT, offsetof(struct lppaca, int_dword.any_int)); 189 DEFINE(LPPACAANYINT, offsetof(struct lppaca, int_dword.any_int));
189 DEFINE(LPPACADECRINT, offsetof(struct lppaca, int_dword.fields.decr_int)); 190 DEFINE(LPPACADECRINT, offsetof(struct lppaca, int_dword.fields.decr_int));
191 DEFINE(LPPACA_PMCINUSE, offsetof(struct lppaca, pmcregs_in_use));
190 DEFINE(LPPACA_DTLIDX, offsetof(struct lppaca, dtl_idx)); 192 DEFINE(LPPACA_DTLIDX, offsetof(struct lppaca, dtl_idx));
193 DEFINE(LPPACA_YIELDCOUNT, offsetof(struct lppaca, yield_count));
191 DEFINE(PACA_DTL_RIDX, offsetof(struct paca_struct, dtl_ridx)); 194 DEFINE(PACA_DTL_RIDX, offsetof(struct paca_struct, dtl_ridx));
192#endif /* CONFIG_PPC_STD_MMU_64 */ 195#endif /* CONFIG_PPC_STD_MMU_64 */
193 DEFINE(PACAEMERGSP, offsetof(struct paca_struct, emergency_sp)); 196 DEFINE(PACAEMERGSP, offsetof(struct paca_struct, emergency_sp));
@@ -198,11 +201,6 @@ int main(void)
198 DEFINE(PACA_USER_TIME, offsetof(struct paca_struct, user_time)); 201 DEFINE(PACA_USER_TIME, offsetof(struct paca_struct, user_time));
199 DEFINE(PACA_SYSTEM_TIME, offsetof(struct paca_struct, system_time)); 202 DEFINE(PACA_SYSTEM_TIME, offsetof(struct paca_struct, system_time));
200 DEFINE(PACA_TRAP_SAVE, offsetof(struct paca_struct, trap_save)); 203 DEFINE(PACA_TRAP_SAVE, offsetof(struct paca_struct, trap_save));
201#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
202 DEFINE(PACA_KVM_SVCPU, offsetof(struct paca_struct, shadow_vcpu));
203 DEFINE(SVCPU_SLB, offsetof(struct kvmppc_book3s_shadow_vcpu, slb));
204 DEFINE(SVCPU_SLB_MAX, offsetof(struct kvmppc_book3s_shadow_vcpu, slb_max));
205#endif
206#endif /* CONFIG_PPC64 */ 204#endif /* CONFIG_PPC64 */
207 205
208 /* RTAS */ 206 /* RTAS */
@@ -397,67 +395,160 @@ int main(void)
397 DEFINE(VCPU_HOST_PID, offsetof(struct kvm_vcpu, arch.host_pid)); 395 DEFINE(VCPU_HOST_PID, offsetof(struct kvm_vcpu, arch.host_pid));
398 DEFINE(VCPU_GPRS, offsetof(struct kvm_vcpu, arch.gpr)); 396 DEFINE(VCPU_GPRS, offsetof(struct kvm_vcpu, arch.gpr));
399 DEFINE(VCPU_VRSAVE, offsetof(struct kvm_vcpu, arch.vrsave)); 397 DEFINE(VCPU_VRSAVE, offsetof(struct kvm_vcpu, arch.vrsave));
398 DEFINE(VCPU_FPRS, offsetof(struct kvm_vcpu, arch.fpr));
399 DEFINE(VCPU_FPSCR, offsetof(struct kvm_vcpu, arch.fpscr));
400#ifdef CONFIG_ALTIVEC
401 DEFINE(VCPU_VRS, offsetof(struct kvm_vcpu, arch.vr));
402 DEFINE(VCPU_VSCR, offsetof(struct kvm_vcpu, arch.vscr));
403#endif
404#ifdef CONFIG_VSX
405 DEFINE(VCPU_VSRS, offsetof(struct kvm_vcpu, arch.vsr));
406#endif
407 DEFINE(VCPU_XER, offsetof(struct kvm_vcpu, arch.xer));
408 DEFINE(VCPU_CTR, offsetof(struct kvm_vcpu, arch.ctr));
409 DEFINE(VCPU_LR, offsetof(struct kvm_vcpu, arch.lr));
410 DEFINE(VCPU_CR, offsetof(struct kvm_vcpu, arch.cr));
411 DEFINE(VCPU_PC, offsetof(struct kvm_vcpu, arch.pc));
412#ifdef CONFIG_KVM_BOOK3S_64_HV
413 DEFINE(VCPU_MSR, offsetof(struct kvm_vcpu, arch.shregs.msr));
414 DEFINE(VCPU_SRR0, offsetof(struct kvm_vcpu, arch.shregs.srr0));
415 DEFINE(VCPU_SRR1, offsetof(struct kvm_vcpu, arch.shregs.srr1));
416 DEFINE(VCPU_SPRG0, offsetof(struct kvm_vcpu, arch.shregs.sprg0));
417 DEFINE(VCPU_SPRG1, offsetof(struct kvm_vcpu, arch.shregs.sprg1));
418 DEFINE(VCPU_SPRG2, offsetof(struct kvm_vcpu, arch.shregs.sprg2));
419 DEFINE(VCPU_SPRG3, offsetof(struct kvm_vcpu, arch.shregs.sprg3));
420#endif
400 DEFINE(VCPU_SPRG4, offsetof(struct kvm_vcpu, arch.sprg4)); 421 DEFINE(VCPU_SPRG4, offsetof(struct kvm_vcpu, arch.sprg4));
401 DEFINE(VCPU_SPRG5, offsetof(struct kvm_vcpu, arch.sprg5)); 422 DEFINE(VCPU_SPRG5, offsetof(struct kvm_vcpu, arch.sprg5));
402 DEFINE(VCPU_SPRG6, offsetof(struct kvm_vcpu, arch.sprg6)); 423 DEFINE(VCPU_SPRG6, offsetof(struct kvm_vcpu, arch.sprg6));
403 DEFINE(VCPU_SPRG7, offsetof(struct kvm_vcpu, arch.sprg7)); 424 DEFINE(VCPU_SPRG7, offsetof(struct kvm_vcpu, arch.sprg7));
404 DEFINE(VCPU_SHADOW_PID, offsetof(struct kvm_vcpu, arch.shadow_pid)); 425 DEFINE(VCPU_SHADOW_PID, offsetof(struct kvm_vcpu, arch.shadow_pid));
426 DEFINE(VCPU_SHADOW_PID1, offsetof(struct kvm_vcpu, arch.shadow_pid1));
405 DEFINE(VCPU_SHARED, offsetof(struct kvm_vcpu, arch.shared)); 427 DEFINE(VCPU_SHARED, offsetof(struct kvm_vcpu, arch.shared));
406 DEFINE(VCPU_SHARED_MSR, offsetof(struct kvm_vcpu_arch_shared, msr)); 428 DEFINE(VCPU_SHARED_MSR, offsetof(struct kvm_vcpu_arch_shared, msr));
429 DEFINE(VCPU_SHADOW_MSR, offsetof(struct kvm_vcpu, arch.shadow_msr));
407 430
408 /* book3s */ 431 /* book3s */
432#ifdef CONFIG_KVM_BOOK3S_64_HV
433 DEFINE(KVM_LPID, offsetof(struct kvm, arch.lpid));
434 DEFINE(KVM_SDR1, offsetof(struct kvm, arch.sdr1));
435 DEFINE(KVM_HOST_LPID, offsetof(struct kvm, arch.host_lpid));
436 DEFINE(KVM_HOST_LPCR, offsetof(struct kvm, arch.host_lpcr));
437 DEFINE(KVM_HOST_SDR1, offsetof(struct kvm, arch.host_sdr1));
438 DEFINE(KVM_TLBIE_LOCK, offsetof(struct kvm, arch.tlbie_lock));
439 DEFINE(KVM_ONLINE_CPUS, offsetof(struct kvm, online_vcpus.counter));
440 DEFINE(KVM_LAST_VCPU, offsetof(struct kvm, arch.last_vcpu));
441 DEFINE(KVM_LPCR, offsetof(struct kvm, arch.lpcr));
442 DEFINE(KVM_RMOR, offsetof(struct kvm, arch.rmor));
443 DEFINE(VCPU_DSISR, offsetof(struct kvm_vcpu, arch.shregs.dsisr));
444 DEFINE(VCPU_DAR, offsetof(struct kvm_vcpu, arch.shregs.dar));
445#endif
409#ifdef CONFIG_PPC_BOOK3S 446#ifdef CONFIG_PPC_BOOK3S
447 DEFINE(VCPU_KVM, offsetof(struct kvm_vcpu, kvm));
448 DEFINE(VCPU_VCPUID, offsetof(struct kvm_vcpu, vcpu_id));
410 DEFINE(VCPU_HOST_RETIP, offsetof(struct kvm_vcpu, arch.host_retip)); 449 DEFINE(VCPU_HOST_RETIP, offsetof(struct kvm_vcpu, arch.host_retip));
411 DEFINE(VCPU_HOST_MSR, offsetof(struct kvm_vcpu, arch.host_msr)); 450 DEFINE(VCPU_HOST_MSR, offsetof(struct kvm_vcpu, arch.host_msr));
412 DEFINE(VCPU_SHADOW_MSR, offsetof(struct kvm_vcpu, arch.shadow_msr)); 451 DEFINE(VCPU_PURR, offsetof(struct kvm_vcpu, arch.purr));
452 DEFINE(VCPU_SPURR, offsetof(struct kvm_vcpu, arch.spurr));
453 DEFINE(VCPU_DSCR, offsetof(struct kvm_vcpu, arch.dscr));
454 DEFINE(VCPU_AMR, offsetof(struct kvm_vcpu, arch.amr));
455 DEFINE(VCPU_UAMOR, offsetof(struct kvm_vcpu, arch.uamor));
456 DEFINE(VCPU_CTRL, offsetof(struct kvm_vcpu, arch.ctrl));
457 DEFINE(VCPU_DABR, offsetof(struct kvm_vcpu, arch.dabr));
413 DEFINE(VCPU_TRAMPOLINE_LOWMEM, offsetof(struct kvm_vcpu, arch.trampoline_lowmem)); 458 DEFINE(VCPU_TRAMPOLINE_LOWMEM, offsetof(struct kvm_vcpu, arch.trampoline_lowmem));
414 DEFINE(VCPU_TRAMPOLINE_ENTER, offsetof(struct kvm_vcpu, arch.trampoline_enter)); 459 DEFINE(VCPU_TRAMPOLINE_ENTER, offsetof(struct kvm_vcpu, arch.trampoline_enter));
415 DEFINE(VCPU_HIGHMEM_HANDLER, offsetof(struct kvm_vcpu, arch.highmem_handler)); 460 DEFINE(VCPU_HIGHMEM_HANDLER, offsetof(struct kvm_vcpu, arch.highmem_handler));
416 DEFINE(VCPU_RMCALL, offsetof(struct kvm_vcpu, arch.rmcall)); 461 DEFINE(VCPU_RMCALL, offsetof(struct kvm_vcpu, arch.rmcall));
417 DEFINE(VCPU_HFLAGS, offsetof(struct kvm_vcpu, arch.hflags)); 462 DEFINE(VCPU_HFLAGS, offsetof(struct kvm_vcpu, arch.hflags));
463 DEFINE(VCPU_DEC, offsetof(struct kvm_vcpu, arch.dec));
464 DEFINE(VCPU_DEC_EXPIRES, offsetof(struct kvm_vcpu, arch.dec_expires));
465 DEFINE(VCPU_PENDING_EXC, offsetof(struct kvm_vcpu, arch.pending_exceptions));
466 DEFINE(VCPU_VPA, offsetof(struct kvm_vcpu, arch.vpa));
467 DEFINE(VCPU_MMCR, offsetof(struct kvm_vcpu, arch.mmcr));
468 DEFINE(VCPU_PMC, offsetof(struct kvm_vcpu, arch.pmc));
469 DEFINE(VCPU_SLB, offsetof(struct kvm_vcpu, arch.slb));
470 DEFINE(VCPU_SLB_MAX, offsetof(struct kvm_vcpu, arch.slb_max));
471 DEFINE(VCPU_SLB_NR, offsetof(struct kvm_vcpu, arch.slb_nr));
472 DEFINE(VCPU_LAST_CPU, offsetof(struct kvm_vcpu, arch.last_cpu));
473 DEFINE(VCPU_FAULT_DSISR, offsetof(struct kvm_vcpu, arch.fault_dsisr));
474 DEFINE(VCPU_FAULT_DAR, offsetof(struct kvm_vcpu, arch.fault_dar));
475 DEFINE(VCPU_LAST_INST, offsetof(struct kvm_vcpu, arch.last_inst));
476 DEFINE(VCPU_TRAP, offsetof(struct kvm_vcpu, arch.trap));
477 DEFINE(VCPU_PTID, offsetof(struct kvm_vcpu, arch.ptid));
478 DEFINE(VCORE_ENTRY_EXIT, offsetof(struct kvmppc_vcore, entry_exit_count));
479 DEFINE(VCORE_NAP_COUNT, offsetof(struct kvmppc_vcore, nap_count));
480 DEFINE(VCORE_IN_GUEST, offsetof(struct kvmppc_vcore, in_guest));
418 DEFINE(VCPU_SVCPU, offsetof(struct kvmppc_vcpu_book3s, shadow_vcpu) - 481 DEFINE(VCPU_SVCPU, offsetof(struct kvmppc_vcpu_book3s, shadow_vcpu) -
419 offsetof(struct kvmppc_vcpu_book3s, vcpu)); 482 offsetof(struct kvmppc_vcpu_book3s, vcpu));
420 DEFINE(SVCPU_CR, offsetof(struct kvmppc_book3s_shadow_vcpu, cr)); 483 DEFINE(VCPU_SLB_E, offsetof(struct kvmppc_slb, orige));
421 DEFINE(SVCPU_XER, offsetof(struct kvmppc_book3s_shadow_vcpu, xer)); 484 DEFINE(VCPU_SLB_V, offsetof(struct kvmppc_slb, origv));
422 DEFINE(SVCPU_CTR, offsetof(struct kvmppc_book3s_shadow_vcpu, ctr)); 485 DEFINE(VCPU_SLB_SIZE, sizeof(struct kvmppc_slb));
423 DEFINE(SVCPU_LR, offsetof(struct kvmppc_book3s_shadow_vcpu, lr)); 486
424 DEFINE(SVCPU_PC, offsetof(struct kvmppc_book3s_shadow_vcpu, pc)); 487#ifdef CONFIG_PPC_BOOK3S_64
425 DEFINE(SVCPU_R0, offsetof(struct kvmppc_book3s_shadow_vcpu, gpr[0])); 488#ifdef CONFIG_KVM_BOOK3S_PR
426 DEFINE(SVCPU_R1, offsetof(struct kvmppc_book3s_shadow_vcpu, gpr[1])); 489# define SVCPU_FIELD(x, f) DEFINE(x, offsetof(struct paca_struct, shadow_vcpu.f))
427 DEFINE(SVCPU_R2, offsetof(struct kvmppc_book3s_shadow_vcpu, gpr[2])); 490#else
428 DEFINE(SVCPU_R3, offsetof(struct kvmppc_book3s_shadow_vcpu, gpr[3])); 491# define SVCPU_FIELD(x, f)
429 DEFINE(SVCPU_R4, offsetof(struct kvmppc_book3s_shadow_vcpu, gpr[4])); 492#endif
430 DEFINE(SVCPU_R5, offsetof(struct kvmppc_book3s_shadow_vcpu, gpr[5])); 493# define HSTATE_FIELD(x, f) DEFINE(x, offsetof(struct paca_struct, kvm_hstate.f))
431 DEFINE(SVCPU_R6, offsetof(struct kvmppc_book3s_shadow_vcpu, gpr[6])); 494#else /* 32-bit */
432 DEFINE(SVCPU_R7, offsetof(struct kvmppc_book3s_shadow_vcpu, gpr[7])); 495# define SVCPU_FIELD(x, f) DEFINE(x, offsetof(struct kvmppc_book3s_shadow_vcpu, f))
433 DEFINE(SVCPU_R8, offsetof(struct kvmppc_book3s_shadow_vcpu, gpr[8])); 496# define HSTATE_FIELD(x, f) DEFINE(x, offsetof(struct kvmppc_book3s_shadow_vcpu, hstate.f))
434 DEFINE(SVCPU_R9, offsetof(struct kvmppc_book3s_shadow_vcpu, gpr[9])); 497#endif
435 DEFINE(SVCPU_R10, offsetof(struct kvmppc_book3s_shadow_vcpu, gpr[10])); 498
436 DEFINE(SVCPU_R11, offsetof(struct kvmppc_book3s_shadow_vcpu, gpr[11])); 499 SVCPU_FIELD(SVCPU_CR, cr);
437 DEFINE(SVCPU_R12, offsetof(struct kvmppc_book3s_shadow_vcpu, gpr[12])); 500 SVCPU_FIELD(SVCPU_XER, xer);
438 DEFINE(SVCPU_R13, offsetof(struct kvmppc_book3s_shadow_vcpu, gpr[13])); 501 SVCPU_FIELD(SVCPU_CTR, ctr);
439 DEFINE(SVCPU_HOST_R1, offsetof(struct kvmppc_book3s_shadow_vcpu, host_r1)); 502 SVCPU_FIELD(SVCPU_LR, lr);
440 DEFINE(SVCPU_HOST_R2, offsetof(struct kvmppc_book3s_shadow_vcpu, host_r2)); 503 SVCPU_FIELD(SVCPU_PC, pc);
441 DEFINE(SVCPU_VMHANDLER, offsetof(struct kvmppc_book3s_shadow_vcpu, 504 SVCPU_FIELD(SVCPU_R0, gpr[0]);
442 vmhandler)); 505 SVCPU_FIELD(SVCPU_R1, gpr[1]);
443 DEFINE(SVCPU_SCRATCH0, offsetof(struct kvmppc_book3s_shadow_vcpu, 506 SVCPU_FIELD(SVCPU_R2, gpr[2]);
444 scratch0)); 507 SVCPU_FIELD(SVCPU_R3, gpr[3]);
445 DEFINE(SVCPU_SCRATCH1, offsetof(struct kvmppc_book3s_shadow_vcpu, 508 SVCPU_FIELD(SVCPU_R4, gpr[4]);
446 scratch1)); 509 SVCPU_FIELD(SVCPU_R5, gpr[5]);
447 DEFINE(SVCPU_IN_GUEST, offsetof(struct kvmppc_book3s_shadow_vcpu, 510 SVCPU_FIELD(SVCPU_R6, gpr[6]);
448 in_guest)); 511 SVCPU_FIELD(SVCPU_R7, gpr[7]);
449 DEFINE(SVCPU_FAULT_DSISR, offsetof(struct kvmppc_book3s_shadow_vcpu, 512 SVCPU_FIELD(SVCPU_R8, gpr[8]);
450 fault_dsisr)); 513 SVCPU_FIELD(SVCPU_R9, gpr[9]);
451 DEFINE(SVCPU_FAULT_DAR, offsetof(struct kvmppc_book3s_shadow_vcpu, 514 SVCPU_FIELD(SVCPU_R10, gpr[10]);
452 fault_dar)); 515 SVCPU_FIELD(SVCPU_R11, gpr[11]);
453 DEFINE(SVCPU_LAST_INST, offsetof(struct kvmppc_book3s_shadow_vcpu, 516 SVCPU_FIELD(SVCPU_R12, gpr[12]);
454 last_inst)); 517 SVCPU_FIELD(SVCPU_R13, gpr[13]);
455 DEFINE(SVCPU_SHADOW_SRR1, offsetof(struct kvmppc_book3s_shadow_vcpu, 518 SVCPU_FIELD(SVCPU_FAULT_DSISR, fault_dsisr);
456 shadow_srr1)); 519 SVCPU_FIELD(SVCPU_FAULT_DAR, fault_dar);
520 SVCPU_FIELD(SVCPU_LAST_INST, last_inst);
521 SVCPU_FIELD(SVCPU_SHADOW_SRR1, shadow_srr1);
457#ifdef CONFIG_PPC_BOOK3S_32 522#ifdef CONFIG_PPC_BOOK3S_32
458 DEFINE(SVCPU_SR, offsetof(struct kvmppc_book3s_shadow_vcpu, sr)); 523 SVCPU_FIELD(SVCPU_SR, sr);
459#endif 524#endif
460#else 525#ifdef CONFIG_PPC64
526 SVCPU_FIELD(SVCPU_SLB, slb);
527 SVCPU_FIELD(SVCPU_SLB_MAX, slb_max);
528#endif
529
530 HSTATE_FIELD(HSTATE_HOST_R1, host_r1);
531 HSTATE_FIELD(HSTATE_HOST_R2, host_r2);
532 HSTATE_FIELD(HSTATE_HOST_MSR, host_msr);
533 HSTATE_FIELD(HSTATE_VMHANDLER, vmhandler);
534 HSTATE_FIELD(HSTATE_SCRATCH0, scratch0);
535 HSTATE_FIELD(HSTATE_SCRATCH1, scratch1);
536 HSTATE_FIELD(HSTATE_IN_GUEST, in_guest);
537
538#ifdef CONFIG_KVM_BOOK3S_64_HV
539 HSTATE_FIELD(HSTATE_KVM_VCPU, kvm_vcpu);
540 HSTATE_FIELD(HSTATE_KVM_VCORE, kvm_vcore);
541 HSTATE_FIELD(HSTATE_XICS_PHYS, xics_phys);
542 HSTATE_FIELD(HSTATE_MMCR, host_mmcr);
543 HSTATE_FIELD(HSTATE_PMC, host_pmc);
544 HSTATE_FIELD(HSTATE_PURR, host_purr);
545 HSTATE_FIELD(HSTATE_SPURR, host_spurr);
546 HSTATE_FIELD(HSTATE_DSCR, host_dscr);
547 HSTATE_FIELD(HSTATE_DABR, dabr);
548 HSTATE_FIELD(HSTATE_DECEXP, dec_expires);
549#endif /* CONFIG_KVM_BOOK3S_64_HV */
550
551#else /* CONFIG_PPC_BOOK3S */
461 DEFINE(VCPU_CR, offsetof(struct kvm_vcpu, arch.cr)); 552 DEFINE(VCPU_CR, offsetof(struct kvm_vcpu, arch.cr));
462 DEFINE(VCPU_XER, offsetof(struct kvm_vcpu, arch.xer)); 553 DEFINE(VCPU_XER, offsetof(struct kvm_vcpu, arch.xer));
463 DEFINE(VCPU_LR, offsetof(struct kvm_vcpu, arch.lr)); 554 DEFINE(VCPU_LR, offsetof(struct kvm_vcpu, arch.lr));
@@ -467,7 +558,7 @@ int main(void)
467 DEFINE(VCPU_FAULT_DEAR, offsetof(struct kvm_vcpu, arch.fault_dear)); 558 DEFINE(VCPU_FAULT_DEAR, offsetof(struct kvm_vcpu, arch.fault_dear));
468 DEFINE(VCPU_FAULT_ESR, offsetof(struct kvm_vcpu, arch.fault_esr)); 559 DEFINE(VCPU_FAULT_ESR, offsetof(struct kvm_vcpu, arch.fault_esr));
469#endif /* CONFIG_PPC_BOOK3S */ 560#endif /* CONFIG_PPC_BOOK3S */
470#endif 561#endif /* CONFIG_KVM */
471 562
472#ifdef CONFIG_KVM_GUEST 563#ifdef CONFIG_KVM_GUEST
473 DEFINE(KVM_MAGIC_SCRATCH1, offsetof(struct kvm_vcpu_arch_shared, 564 DEFINE(KVM_MAGIC_SCRATCH1, offsetof(struct kvm_vcpu_arch_shared,
@@ -497,6 +588,13 @@ int main(void)
497 DEFINE(TLBCAM_MAS7, offsetof(struct tlbcam, MAS7)); 588 DEFINE(TLBCAM_MAS7, offsetof(struct tlbcam, MAS7));
498#endif 589#endif
499 590
591#if defined(CONFIG_KVM) && defined(CONFIG_SPE)
592 DEFINE(VCPU_EVR, offsetof(struct kvm_vcpu, arch.evr[0]));
593 DEFINE(VCPU_ACC, offsetof(struct kvm_vcpu, arch.acc));
594 DEFINE(VCPU_SPEFSCR, offsetof(struct kvm_vcpu, arch.spefscr));
595 DEFINE(VCPU_HOST_SPEFSCR, offsetof(struct kvm_vcpu, arch.host_spefscr));
596#endif
597
500#ifdef CONFIG_KVM_EXIT_TIMING 598#ifdef CONFIG_KVM_EXIT_TIMING
501 DEFINE(VCPU_TIMING_EXIT_TBU, offsetof(struct kvm_vcpu, 599 DEFINE(VCPU_TIMING_EXIT_TBU, offsetof(struct kvm_vcpu,
502 arch.timing_exit.tv32.tbu)); 600 arch.timing_exit.tv32.tbu));
diff --git a/arch/powerpc/kernel/cpu_setup_power7.S b/arch/powerpc/kernel/cpu_setup_power7.S
index 4f9a93fcfe07..76797c5105d6 100644
--- a/arch/powerpc/kernel/cpu_setup_power7.S
+++ b/arch/powerpc/kernel/cpu_setup_power7.S
@@ -45,12 +45,12 @@ _GLOBAL(__restore_cpu_power7)
45 blr 45 blr
46 46
47__init_hvmode_206: 47__init_hvmode_206:
48 /* Disable CPU_FTR_HVMODE_206 and exit if MSR:HV is not set */ 48 /* Disable CPU_FTR_HVMODE and exit if MSR:HV is not set */
49 mfmsr r3 49 mfmsr r3
50 rldicl. r0,r3,4,63 50 rldicl. r0,r3,4,63
51 bnelr 51 bnelr
52 ld r5,CPU_SPEC_FEATURES(r4) 52 ld r5,CPU_SPEC_FEATURES(r4)
53 LOAD_REG_IMMEDIATE(r6,CPU_FTR_HVMODE_206) 53 LOAD_REG_IMMEDIATE(r6,CPU_FTR_HVMODE)
54 xor r5,r5,r6 54 xor r5,r5,r6
55 std r5,CPU_SPEC_FEATURES(r4) 55 std r5,CPU_SPEC_FEATURES(r4)
56 blr 56 blr
@@ -61,19 +61,23 @@ __init_LPCR:
61 * LPES = 0b01 (HSRR0/1 used for 0x500) 61 * LPES = 0b01 (HSRR0/1 used for 0x500)
62 * PECE = 0b111 62 * PECE = 0b111
63 * DPFD = 4 63 * DPFD = 4
64 * HDICE = 0
65 * VC = 0b100 (VPM0=1, VPM1=0, ISL=0)
66 * VRMASD = 0b10000 (L=1, LP=00)
64 * 67 *
65 * Other bits untouched for now 68 * Other bits untouched for now
66 */ 69 */
67 mfspr r3,SPRN_LPCR 70 mfspr r3,SPRN_LPCR
68 ori r3,r3,(LPCR_LPES0|LPCR_LPES1) 71 li r5,1
69 xori r3,r3, LPCR_LPES0 72 rldimi r3,r5, LPCR_LPES_SH, 64-LPCR_LPES_SH-2
70 ori r3,r3,(LPCR_PECE0|LPCR_PECE1|LPCR_PECE2) 73 ori r3,r3,(LPCR_PECE0|LPCR_PECE1|LPCR_PECE2)
71 li r5,7
72 sldi r5,r5,LPCR_DPFD_SH
73 andc r3,r3,r5
74 li r5,4 74 li r5,4
75 sldi r5,r5,LPCR_DPFD_SH 75 rldimi r3,r5, LPCR_DPFD_SH, 64-LPCR_DPFD_SH-3
76 or r3,r3,r5 76 clrrdi r3,r3,1 /* clear HDICE */
77 li r5,4
78 rldimi r3,r5, LPCR_VC_SH, 0
79 li r5,0x10
80 rldimi r3,r5, LPCR_VRMASD_SH, 64-LPCR_VRMASD_SH-5
77 mtspr SPRN_LPCR,r3 81 mtspr SPRN_LPCR,r3
78 isync 82 isync
79 blr 83 blr
diff --git a/arch/powerpc/kernel/cpu_setup_ppc970.S b/arch/powerpc/kernel/cpu_setup_ppc970.S
index 27f2507279d8..12fac8df01c5 100644
--- a/arch/powerpc/kernel/cpu_setup_ppc970.S
+++ b/arch/powerpc/kernel/cpu_setup_ppc970.S
@@ -76,7 +76,7 @@ _GLOBAL(__setup_cpu_ppc970)
76 /* Do nothing if not running in HV mode */ 76 /* Do nothing if not running in HV mode */
77 mfmsr r0 77 mfmsr r0
78 rldicl. r0,r0,4,63 78 rldicl. r0,r0,4,63
79 beqlr 79 beq no_hv_mode
80 80
81 mfspr r0,SPRN_HID0 81 mfspr r0,SPRN_HID0
82 li r11,5 /* clear DOZE and SLEEP */ 82 li r11,5 /* clear DOZE and SLEEP */
@@ -90,7 +90,7 @@ _GLOBAL(__setup_cpu_ppc970MP)
90 /* Do nothing if not running in HV mode */ 90 /* Do nothing if not running in HV mode */
91 mfmsr r0 91 mfmsr r0
92 rldicl. r0,r0,4,63 92 rldicl. r0,r0,4,63
93 beqlr 93 beq no_hv_mode
94 94
95 mfspr r0,SPRN_HID0 95 mfspr r0,SPRN_HID0
96 li r11,0x15 /* clear DOZE and SLEEP */ 96 li r11,0x15 /* clear DOZE and SLEEP */
@@ -109,6 +109,14 @@ load_hids:
109 sync 109 sync
110 isync 110 isync
111 111
112 /* Try to set LPES = 01 in HID4 */
113 mfspr r0,SPRN_HID4
114 clrldi r0,r0,1 /* clear LPES0 */
115 ori r0,r0,HID4_LPES1 /* set LPES1 */
116 sync
117 mtspr SPRN_HID4,r0
118 isync
119
112 /* Save away cpu state */ 120 /* Save away cpu state */
113 LOAD_REG_ADDR(r5,cpu_state_storage) 121 LOAD_REG_ADDR(r5,cpu_state_storage)
114 122
@@ -117,11 +125,21 @@ load_hids:
117 std r3,CS_HID0(r5) 125 std r3,CS_HID0(r5)
118 mfspr r3,SPRN_HID1 126 mfspr r3,SPRN_HID1
119 std r3,CS_HID1(r5) 127 std r3,CS_HID1(r5)
120 mfspr r3,SPRN_HID4 128 mfspr r4,SPRN_HID4
121 std r3,CS_HID4(r5) 129 std r4,CS_HID4(r5)
122 mfspr r3,SPRN_HID5 130 mfspr r3,SPRN_HID5
123 std r3,CS_HID5(r5) 131 std r3,CS_HID5(r5)
124 132
133 /* See if we successfully set LPES1 to 1; if not we are in Apple mode */
134 andi. r4,r4,HID4_LPES1
135 bnelr
136
137no_hv_mode:
138 /* Disable CPU_FTR_HVMODE and exit, since we don't have HV mode */
139 ld r5,CPU_SPEC_FEATURES(r4)
140 LOAD_REG_IMMEDIATE(r6,CPU_FTR_HVMODE)
141 andc r5,r5,r6
142 std r5,CPU_SPEC_FEATURES(r4)
125 blr 143 blr
126 144
127/* Called with no MMU context (typically MSR:IR/DR off) to 145/* Called with no MMU context (typically MSR:IR/DR off) to
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index a85f4874cba7..41b02c792aa3 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -40,7 +40,6 @@ __start_interrupts:
40 .globl system_reset_pSeries; 40 .globl system_reset_pSeries;
41system_reset_pSeries: 41system_reset_pSeries:
42 HMT_MEDIUM; 42 HMT_MEDIUM;
43 DO_KVM 0x100;
44 SET_SCRATCH0(r13) 43 SET_SCRATCH0(r13)
45#ifdef CONFIG_PPC_P7_NAP 44#ifdef CONFIG_PPC_P7_NAP
46BEGIN_FTR_SECTION 45BEGIN_FTR_SECTION
@@ -50,82 +49,73 @@ BEGIN_FTR_SECTION
50 * state loss at this time. 49 * state loss at this time.
51 */ 50 */
52 mfspr r13,SPRN_SRR1 51 mfspr r13,SPRN_SRR1
53 rlwinm r13,r13,47-31,30,31 52 rlwinm. r13,r13,47-31,30,31
54 cmpwi cr0,r13,1 53 beq 9f
55 bne 1f 54
56 b .power7_wakeup_noloss 55 /* waking up from powersave (nap) state */
571: cmpwi cr0,r13,2 56 cmpwi cr1,r13,2
58 bne 1f
59 b .power7_wakeup_loss
60 /* Total loss of HV state is fatal, we could try to use the 57 /* Total loss of HV state is fatal, we could try to use the
61 * PIR to locate a PACA, then use an emergency stack etc... 58 * PIR to locate a PACA, then use an emergency stack etc...
62 * but for now, let's just stay stuck here 59 * but for now, let's just stay stuck here
63 */ 60 */
641: cmpwi cr0,r13,3 61 bgt cr1,.
65 beq . 62 GET_PACA(r13)
66END_FTR_SECTION_IFSET(CPU_FTR_HVMODE_206) 63
64#ifdef CONFIG_KVM_BOOK3S_64_HV
65 lbz r0,PACAPROCSTART(r13)
66 cmpwi r0,0x80
67 bne 1f
68 li r0,0
69 stb r0,PACAPROCSTART(r13)
70 b kvm_start_guest
711:
72#endif
73
74 beq cr1,2f
75 b .power7_wakeup_noloss
762: b .power7_wakeup_loss
779:
78END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
67#endif /* CONFIG_PPC_P7_NAP */ 79#endif /* CONFIG_PPC_P7_NAP */
68 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common, EXC_STD) 80 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common, EXC_STD,
81 NOTEST, 0x100)
69 82
70 . = 0x200 83 . = 0x200
71_machine_check_pSeries: 84machine_check_pSeries_1:
72 HMT_MEDIUM 85 /* This is moved out of line as it can be patched by FW, but
73 DO_KVM 0x200 86 * some code path might still want to branch into the original
74 SET_SCRATCH0(r13) 87 * vector
75 EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common, EXC_STD) 88 */
89 b machine_check_pSeries
76 90
77 . = 0x300 91 . = 0x300
78 .globl data_access_pSeries 92 .globl data_access_pSeries
79data_access_pSeries: 93data_access_pSeries:
80 HMT_MEDIUM 94 HMT_MEDIUM
81 DO_KVM 0x300
82 SET_SCRATCH0(r13) 95 SET_SCRATCH0(r13)
96#ifndef CONFIG_POWER4_ONLY
83BEGIN_FTR_SECTION 97BEGIN_FTR_SECTION
84 GET_PACA(r13) 98 b data_access_check_stab
85 std r9,PACA_EXSLB+EX_R9(r13) 99data_access_not_stab:
86 std r10,PACA_EXSLB+EX_R10(r13) 100END_MMU_FTR_SECTION_IFCLR(MMU_FTR_SLB)
87 mfspr r10,SPRN_DAR 101#endif
88 mfspr r9,SPRN_DSISR 102 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, data_access_common, EXC_STD,
89 srdi r10,r10,60 103 KVMTEST_PR, 0x300)
90 rlwimi r10,r9,16,0x20
91 mfcr r9
92 cmpwi r10,0x2c
93 beq do_stab_bolted_pSeries
94 ld r10,PACA_EXSLB+EX_R10(r13)
95 std r11,PACA_EXGEN+EX_R11(r13)
96 ld r11,PACA_EXSLB+EX_R9(r13)
97 std r12,PACA_EXGEN+EX_R12(r13)
98 GET_SCRATCH0(r12)
99 std r10,PACA_EXGEN+EX_R10(r13)
100 std r11,PACA_EXGEN+EX_R9(r13)
101 std r12,PACA_EXGEN+EX_R13(r13)
102 EXCEPTION_PROLOG_PSERIES_1(data_access_common, EXC_STD)
103FTR_SECTION_ELSE
104 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, data_access_common, EXC_STD)
105ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_SLB)
106 104
107 . = 0x380 105 . = 0x380
108 .globl data_access_slb_pSeries 106 .globl data_access_slb_pSeries
109data_access_slb_pSeries: 107data_access_slb_pSeries:
110 HMT_MEDIUM 108 HMT_MEDIUM
111 DO_KVM 0x380
112 SET_SCRATCH0(r13) 109 SET_SCRATCH0(r13)
113 GET_PACA(r13) 110 EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST_PR, 0x380)
114 std r3,PACA_EXSLB+EX_R3(r13) 111 std r3,PACA_EXSLB+EX_R3(r13)
115 mfspr r3,SPRN_DAR 112 mfspr r3,SPRN_DAR
116 std r9,PACA_EXSLB+EX_R9(r13) /* save r9 - r12 */
117 mfcr r9
118#ifdef __DISABLED__ 113#ifdef __DISABLED__
119 /* Keep that around for when we re-implement dynamic VSIDs */ 114 /* Keep that around for when we re-implement dynamic VSIDs */
120 cmpdi r3,0 115 cmpdi r3,0
121 bge slb_miss_user_pseries 116 bge slb_miss_user_pseries
122#endif /* __DISABLED__ */ 117#endif /* __DISABLED__ */
123 std r10,PACA_EXSLB+EX_R10(r13) 118 mfspr r12,SPRN_SRR1
124 std r11,PACA_EXSLB+EX_R11(r13)
125 std r12,PACA_EXSLB+EX_R12(r13)
126 GET_SCRATCH0(r10)
127 std r10,PACA_EXSLB+EX_R13(r13)
128 mfspr r12,SPRN_SRR1 /* and SRR1 */
129#ifndef CONFIG_RELOCATABLE 119#ifndef CONFIG_RELOCATABLE
130 b .slb_miss_realmode 120 b .slb_miss_realmode
131#else 121#else
@@ -147,24 +137,16 @@ data_access_slb_pSeries:
147 .globl instruction_access_slb_pSeries 137 .globl instruction_access_slb_pSeries
148instruction_access_slb_pSeries: 138instruction_access_slb_pSeries:
149 HMT_MEDIUM 139 HMT_MEDIUM
150 DO_KVM 0x480
151 SET_SCRATCH0(r13) 140 SET_SCRATCH0(r13)
152 GET_PACA(r13) 141 EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST_PR, 0x480)
153 std r3,PACA_EXSLB+EX_R3(r13) 142 std r3,PACA_EXSLB+EX_R3(r13)
154 mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */ 143 mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */
155 std r9,PACA_EXSLB+EX_R9(r13) /* save r9 - r12 */
156 mfcr r9
157#ifdef __DISABLED__ 144#ifdef __DISABLED__
158 /* Keep that around for when we re-implement dynamic VSIDs */ 145 /* Keep that around for when we re-implement dynamic VSIDs */
159 cmpdi r3,0 146 cmpdi r3,0
160 bge slb_miss_user_pseries 147 bge slb_miss_user_pseries
161#endif /* __DISABLED__ */ 148#endif /* __DISABLED__ */
162 std r10,PACA_EXSLB+EX_R10(r13) 149 mfspr r12,SPRN_SRR1
163 std r11,PACA_EXSLB+EX_R11(r13)
164 std r12,PACA_EXSLB+EX_R12(r13)
165 GET_SCRATCH0(r10)
166 std r10,PACA_EXSLB+EX_R13(r13)
167 mfspr r12,SPRN_SRR1 /* and SRR1 */
168#ifndef CONFIG_RELOCATABLE 150#ifndef CONFIG_RELOCATABLE
169 b .slb_miss_realmode 151 b .slb_miss_realmode
170#else 152#else
@@ -184,26 +166,46 @@ instruction_access_slb_pSeries:
184hardware_interrupt_pSeries: 166hardware_interrupt_pSeries:
185hardware_interrupt_hv: 167hardware_interrupt_hv:
186 BEGIN_FTR_SECTION 168 BEGIN_FTR_SECTION
187 _MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt, EXC_STD) 169 _MASKABLE_EXCEPTION_PSERIES(0x502, hardware_interrupt,
170 EXC_HV, SOFTEN_TEST_HV)
171 KVM_HANDLER(PACA_EXGEN, EXC_HV, 0x502)
188 FTR_SECTION_ELSE 172 FTR_SECTION_ELSE
189 _MASKABLE_EXCEPTION_PSERIES(0x502, hardware_interrupt, EXC_HV) 173 _MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt,
190 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_HVMODE_206) 174 EXC_STD, SOFTEN_TEST_HV_201)
175 KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x500)
176 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
191 177
192 STD_EXCEPTION_PSERIES(0x600, 0x600, alignment) 178 STD_EXCEPTION_PSERIES(0x600, 0x600, alignment)
179 KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x600)
180
193 STD_EXCEPTION_PSERIES(0x700, 0x700, program_check) 181 STD_EXCEPTION_PSERIES(0x700, 0x700, program_check)
182 KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x700)
183
194 STD_EXCEPTION_PSERIES(0x800, 0x800, fp_unavailable) 184 STD_EXCEPTION_PSERIES(0x800, 0x800, fp_unavailable)
185 KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x800)
195 186
196 MASKABLE_EXCEPTION_PSERIES(0x900, 0x900, decrementer) 187 MASKABLE_EXCEPTION_PSERIES(0x900, 0x900, decrementer)
197 MASKABLE_EXCEPTION_HV(0x980, 0x980, decrementer) 188 MASKABLE_EXCEPTION_HV(0x980, 0x982, decrementer)
198 189
199 STD_EXCEPTION_PSERIES(0xa00, 0xa00, trap_0a) 190 STD_EXCEPTION_PSERIES(0xa00, 0xa00, trap_0a)
191 KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xa00)
192
200 STD_EXCEPTION_PSERIES(0xb00, 0xb00, trap_0b) 193 STD_EXCEPTION_PSERIES(0xb00, 0xb00, trap_0b)
194 KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xb00)
201 195
202 . = 0xc00 196 . = 0xc00
203 .globl system_call_pSeries 197 .globl system_call_pSeries
204system_call_pSeries: 198system_call_pSeries:
205 HMT_MEDIUM 199 HMT_MEDIUM
206 DO_KVM 0xc00 200#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
201 SET_SCRATCH0(r13)
202 GET_PACA(r13)
203 std r9,PACA_EXGEN+EX_R9(r13)
204 std r10,PACA_EXGEN+EX_R10(r13)
205 mfcr r9
206 KVMTEST(0xc00)
207 GET_SCRATCH0(r13)
208#endif
207BEGIN_FTR_SECTION 209BEGIN_FTR_SECTION
208 cmpdi r0,0x1ebe 210 cmpdi r0,0x1ebe
209 beq- 1f 211 beq- 1f
@@ -220,6 +222,8 @@ END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE)
220 rfid 222 rfid
221 b . /* prevent speculative execution */ 223 b . /* prevent speculative execution */
222 224
225 KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xc00)
226
223/* Fast LE/BE switch system call */ 227/* Fast LE/BE switch system call */
2241: mfspr r12,SPRN_SRR1 2281: mfspr r12,SPRN_SRR1
225 xori r12,r12,MSR_LE 229 xori r12,r12,MSR_LE
@@ -228,6 +232,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE)
228 b . 232 b .
229 233
230 STD_EXCEPTION_PSERIES(0xd00, 0xd00, single_step) 234 STD_EXCEPTION_PSERIES(0xd00, 0xd00, single_step)
235 KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xd00)
231 236
232 /* At 0xe??? we have a bunch of hypervisor exceptions, we branch 237 /* At 0xe??? we have a bunch of hypervisor exceptions, we branch
233 * out of line to handle them 238 * out of line to handle them
@@ -262,30 +267,93 @@ vsx_unavailable_pSeries_1:
262 267
263#ifdef CONFIG_CBE_RAS 268#ifdef CONFIG_CBE_RAS
264 STD_EXCEPTION_HV(0x1200, 0x1202, cbe_system_error) 269 STD_EXCEPTION_HV(0x1200, 0x1202, cbe_system_error)
270 KVM_HANDLER_PR_SKIP(PACA_EXGEN, EXC_HV, 0x1202)
265#endif /* CONFIG_CBE_RAS */ 271#endif /* CONFIG_CBE_RAS */
272
266 STD_EXCEPTION_PSERIES(0x1300, 0x1300, instruction_breakpoint) 273 STD_EXCEPTION_PSERIES(0x1300, 0x1300, instruction_breakpoint)
274 KVM_HANDLER_PR_SKIP(PACA_EXGEN, EXC_STD, 0x1300)
275
267#ifdef CONFIG_CBE_RAS 276#ifdef CONFIG_CBE_RAS
268 STD_EXCEPTION_HV(0x1600, 0x1602, cbe_maintenance) 277 STD_EXCEPTION_HV(0x1600, 0x1602, cbe_maintenance)
278 KVM_HANDLER_PR_SKIP(PACA_EXGEN, EXC_HV, 0x1602)
269#endif /* CONFIG_CBE_RAS */ 279#endif /* CONFIG_CBE_RAS */
280
270 STD_EXCEPTION_PSERIES(0x1700, 0x1700, altivec_assist) 281 STD_EXCEPTION_PSERIES(0x1700, 0x1700, altivec_assist)
282 KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x1700)
283
271#ifdef CONFIG_CBE_RAS 284#ifdef CONFIG_CBE_RAS
272 STD_EXCEPTION_HV(0x1800, 0x1802, cbe_thermal) 285 STD_EXCEPTION_HV(0x1800, 0x1802, cbe_thermal)
286 KVM_HANDLER_PR_SKIP(PACA_EXGEN, EXC_HV, 0x1802)
273#endif /* CONFIG_CBE_RAS */ 287#endif /* CONFIG_CBE_RAS */
274 288
275 . = 0x3000 289 . = 0x3000
276 290
277/*** Out of line interrupts support ***/ 291/*** Out of line interrupts support ***/
278 292
293 /* moved from 0x200 */
294machine_check_pSeries:
295 .globl machine_check_fwnmi
296machine_check_fwnmi:
297 HMT_MEDIUM
298 SET_SCRATCH0(r13) /* save r13 */
299 EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common,
300 EXC_STD, KVMTEST, 0x200)
301 KVM_HANDLER_SKIP(PACA_EXMC, EXC_STD, 0x200)
302
303#ifndef CONFIG_POWER4_ONLY
304 /* moved from 0x300 */
305data_access_check_stab:
306 GET_PACA(r13)
307 std r9,PACA_EXSLB+EX_R9(r13)
308 std r10,PACA_EXSLB+EX_R10(r13)
309 mfspr r10,SPRN_DAR
310 mfspr r9,SPRN_DSISR
311 srdi r10,r10,60
312 rlwimi r10,r9,16,0x20
313#ifdef CONFIG_KVM_BOOK3S_PR
314 lbz r9,HSTATE_IN_GUEST(r13)
315 rlwimi r10,r9,8,0x300
316#endif
317 mfcr r9
318 cmpwi r10,0x2c
319 beq do_stab_bolted_pSeries
320 mtcrf 0x80,r9
321 ld r9,PACA_EXSLB+EX_R9(r13)
322 ld r10,PACA_EXSLB+EX_R10(r13)
323 b data_access_not_stab
324do_stab_bolted_pSeries:
325 std r11,PACA_EXSLB+EX_R11(r13)
326 std r12,PACA_EXSLB+EX_R12(r13)
327 GET_SCRATCH0(r10)
328 std r10,PACA_EXSLB+EX_R13(r13)
329 EXCEPTION_PROLOG_PSERIES_1(.do_stab_bolted, EXC_STD)
330#endif /* CONFIG_POWER4_ONLY */
331
332 KVM_HANDLER_PR_SKIP(PACA_EXGEN, EXC_STD, 0x300)
333 KVM_HANDLER_PR_SKIP(PACA_EXSLB, EXC_STD, 0x380)
334 KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x400)
335 KVM_HANDLER_PR(PACA_EXSLB, EXC_STD, 0x480)
336 KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x900)
337 KVM_HANDLER(PACA_EXGEN, EXC_HV, 0x982)
338
339 .align 7
279 /* moved from 0xe00 */ 340 /* moved from 0xe00 */
280 STD_EXCEPTION_HV(., 0xe00, h_data_storage) 341 STD_EXCEPTION_HV(., 0xe02, h_data_storage)
281 STD_EXCEPTION_HV(., 0xe20, h_instr_storage) 342 KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0xe02)
282 STD_EXCEPTION_HV(., 0xe40, emulation_assist) 343 STD_EXCEPTION_HV(., 0xe22, h_instr_storage)
283 STD_EXCEPTION_HV(., 0xe60, hmi_exception) /* need to flush cache ? */ 344 KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe22)
345 STD_EXCEPTION_HV(., 0xe42, emulation_assist)
346 KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe42)
347 STD_EXCEPTION_HV(., 0xe62, hmi_exception) /* need to flush cache ? */
348 KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe62)
284 349
285 /* moved from 0xf00 */ 350 /* moved from 0xf00 */
286 STD_EXCEPTION_PSERIES(., 0xf00, performance_monitor) 351 STD_EXCEPTION_PSERIES(., 0xf00, performance_monitor)
352 KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xf00)
287 STD_EXCEPTION_PSERIES(., 0xf20, altivec_unavailable) 353 STD_EXCEPTION_PSERIES(., 0xf20, altivec_unavailable)
354 KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xf20)
288 STD_EXCEPTION_PSERIES(., 0xf40, vsx_unavailable) 355 STD_EXCEPTION_PSERIES(., 0xf40, vsx_unavailable)
356 KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xf40)
289 357
290/* 358/*
291 * An interrupt came in while soft-disabled; clear EE in SRR1, 359 * An interrupt came in while soft-disabled; clear EE in SRR1,
@@ -317,14 +385,6 @@ masked_Hinterrupt:
317 hrfid 385 hrfid
318 b . 386 b .
319 387
320 .align 7
321do_stab_bolted_pSeries:
322 std r11,PACA_EXSLB+EX_R11(r13)
323 std r12,PACA_EXSLB+EX_R12(r13)
324 GET_SCRATCH0(r10)
325 std r10,PACA_EXSLB+EX_R13(r13)
326 EXCEPTION_PROLOG_PSERIES_1(.do_stab_bolted, EXC_STD)
327
328#ifdef CONFIG_PPC_PSERIES 388#ifdef CONFIG_PPC_PSERIES
329/* 389/*
330 * Vectors for the FWNMI option. Share common code. 390 * Vectors for the FWNMI option. Share common code.
@@ -334,14 +394,8 @@ do_stab_bolted_pSeries:
334system_reset_fwnmi: 394system_reset_fwnmi:
335 HMT_MEDIUM 395 HMT_MEDIUM
336 SET_SCRATCH0(r13) /* save r13 */ 396 SET_SCRATCH0(r13) /* save r13 */
337 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common, EXC_STD) 397 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common, EXC_STD,
338 398 NOTEST, 0x100)
339 .globl machine_check_fwnmi
340 .align 7
341machine_check_fwnmi:
342 HMT_MEDIUM
343 SET_SCRATCH0(r13) /* save r13 */
344 EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common, EXC_STD)
345 399
346#endif /* CONFIG_PPC_PSERIES */ 400#endif /* CONFIG_PPC_PSERIES */
347 401
@@ -376,7 +430,11 @@ slb_miss_user_pseries:
376/* KVM's trampoline code needs to be close to the interrupt handlers */ 430/* KVM's trampoline code needs to be close to the interrupt handlers */
377 431
378#ifdef CONFIG_KVM_BOOK3S_64_HANDLER 432#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
433#ifdef CONFIG_KVM_BOOK3S_PR
379#include "../kvm/book3s_rmhandlers.S" 434#include "../kvm/book3s_rmhandlers.S"
435#else
436#include "../kvm/book3s_hv_rmhandlers.S"
437#endif
380#endif 438#endif
381 439
382 .align 7 440 .align 7
diff --git a/arch/powerpc/kernel/head_fsl_booke.S b/arch/powerpc/kernel/head_fsl_booke.S
index 5ecf54cfa7d4..fe37dd0dfd17 100644
--- a/arch/powerpc/kernel/head_fsl_booke.S
+++ b/arch/powerpc/kernel/head_fsl_booke.S
@@ -656,7 +656,7 @@ load_up_spe:
656 cmpi 0,r4,0 656 cmpi 0,r4,0
657 beq 1f 657 beq 1f
658 addi r4,r4,THREAD /* want THREAD of last_task_used_spe */ 658 addi r4,r4,THREAD /* want THREAD of last_task_used_spe */
659 SAVE_32EVRS(0,r10,r4) 659 SAVE_32EVRS(0,r10,r4,THREAD_EVR0)
660 evxor evr10, evr10, evr10 /* clear out evr10 */ 660 evxor evr10, evr10, evr10 /* clear out evr10 */
661 evmwumiaa evr10, evr10, evr10 /* evr10 <- ACC = 0 * 0 + ACC */ 661 evmwumiaa evr10, evr10, evr10 /* evr10 <- ACC = 0 * 0 + ACC */
662 li r5,THREAD_ACC 662 li r5,THREAD_ACC
@@ -676,7 +676,7 @@ load_up_spe:
676 stw r4,THREAD_USED_SPE(r5) 676 stw r4,THREAD_USED_SPE(r5)
677 evlddx evr4,r10,r5 677 evlddx evr4,r10,r5
678 evmra evr4,evr4 678 evmra evr4,evr4
679 REST_32EVRS(0,r10,r5) 679 REST_32EVRS(0,r10,r5,THREAD_EVR0)
680#ifndef CONFIG_SMP 680#ifndef CONFIG_SMP
681 subi r4,r5,THREAD 681 subi r4,r5,THREAD
682 stw r4,last_task_used_spe@l(r3) 682 stw r4,last_task_used_spe@l(r3)
@@ -787,13 +787,11 @@ _GLOBAL(giveup_spe)
787 addi r3,r3,THREAD /* want THREAD of task */ 787 addi r3,r3,THREAD /* want THREAD of task */
788 lwz r5,PT_REGS(r3) 788 lwz r5,PT_REGS(r3)
789 cmpi 0,r5,0 789 cmpi 0,r5,0
790 SAVE_32EVRS(0, r4, r3) 790 SAVE_32EVRS(0, r4, r3, THREAD_EVR0)
791 evxor evr6, evr6, evr6 /* clear out evr6 */ 791 evxor evr6, evr6, evr6 /* clear out evr6 */
792 evmwumiaa evr6, evr6, evr6 /* evr6 <- ACC = 0 * 0 + ACC */ 792 evmwumiaa evr6, evr6, evr6 /* evr6 <- ACC = 0 * 0 + ACC */
793 li r4,THREAD_ACC 793 li r4,THREAD_ACC
794 evstddx evr6, r4, r3 /* save off accumulator */ 794 evstddx evr6, r4, r3 /* save off accumulator */
795 mfspr r6,SPRN_SPEFSCR
796 stw r6,THREAD_SPEFSCR(r3) /* save spefscr register value */
797 beq 1f 795 beq 1f
798 lwz r4,_MSR-STACK_FRAME_OVERHEAD(r5) 796 lwz r4,_MSR-STACK_FRAME_OVERHEAD(r5)
799 lis r3,MSR_SPE@h 797 lis r3,MSR_SPE@h
diff --git a/arch/powerpc/kernel/idle_power7.S b/arch/powerpc/kernel/idle_power7.S
index f8f0bc7f1d4f..3a70845a51c7 100644
--- a/arch/powerpc/kernel/idle_power7.S
+++ b/arch/powerpc/kernel/idle_power7.S
@@ -73,7 +73,6 @@ _GLOBAL(power7_idle)
73 b . 73 b .
74 74
75_GLOBAL(power7_wakeup_loss) 75_GLOBAL(power7_wakeup_loss)
76 GET_PACA(r13)
77 ld r1,PACAR1(r13) 76 ld r1,PACAR1(r13)
78 REST_NVGPRS(r1) 77 REST_NVGPRS(r1)
79 REST_GPR(2, r1) 78 REST_GPR(2, r1)
@@ -87,7 +86,6 @@ _GLOBAL(power7_wakeup_loss)
87 rfid 86 rfid
88 87
89_GLOBAL(power7_wakeup_noloss) 88_GLOBAL(power7_wakeup_noloss)
90 GET_PACA(r13)
91 ld r1,PACAR1(r13) 89 ld r1,PACAR1(r13)
92 ld r4,_MSR(r1) 90 ld r4,_MSR(r1)
93 ld r5,_NIP(r1) 91 ld r5,_NIP(r1)
diff --git a/arch/powerpc/kernel/paca.c b/arch/powerpc/kernel/paca.c
index efeb88184182..0a5a899846bb 100644
--- a/arch/powerpc/kernel/paca.c
+++ b/arch/powerpc/kernel/paca.c
@@ -167,7 +167,7 @@ void setup_paca(struct paca_struct *new_paca)
167 * if we do a GET_PACA() before the feature fixups have been 167 * if we do a GET_PACA() before the feature fixups have been
168 * applied 168 * applied
169 */ 169 */
170 if (cpu_has_feature(CPU_FTR_HVMODE_206)) 170 if (cpu_has_feature(CPU_FTR_HVMODE))
171 mtspr(SPRN_SPRG_HPACA, local_paca); 171 mtspr(SPRN_SPRG_HPACA, local_paca);
172#endif 172#endif
173 mtspr(SPRN_SPRG_PACA, local_paca); 173 mtspr(SPRN_SPRG_PACA, local_paca);
diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index 91e52df3d81d..ec2d0edeb134 100644
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -96,6 +96,7 @@ void flush_fp_to_thread(struct task_struct *tsk)
96 preempt_enable(); 96 preempt_enable();
97 } 97 }
98} 98}
99EXPORT_SYMBOL_GPL(flush_fp_to_thread);
99 100
100void enable_kernel_fp(void) 101void enable_kernel_fp(void)
101{ 102{
@@ -145,6 +146,7 @@ void flush_altivec_to_thread(struct task_struct *tsk)
145 preempt_enable(); 146 preempt_enable();
146 } 147 }
147} 148}
149EXPORT_SYMBOL_GPL(flush_altivec_to_thread);
148#endif /* CONFIG_ALTIVEC */ 150#endif /* CONFIG_ALTIVEC */
149 151
150#ifdef CONFIG_VSX 152#ifdef CONFIG_VSX
@@ -186,6 +188,7 @@ void flush_vsx_to_thread(struct task_struct *tsk)
186 preempt_enable(); 188 preempt_enable();
187 } 189 }
188} 190}
191EXPORT_SYMBOL_GPL(flush_vsx_to_thread);
189#endif /* CONFIG_VSX */ 192#endif /* CONFIG_VSX */
190 193
191#ifdef CONFIG_SPE 194#ifdef CONFIG_SPE
@@ -213,6 +216,7 @@ void flush_spe_to_thread(struct task_struct *tsk)
213#ifdef CONFIG_SMP 216#ifdef CONFIG_SMP
214 BUG_ON(tsk != current); 217 BUG_ON(tsk != current);
215#endif 218#endif
219 tsk->thread.spefscr = mfspr(SPRN_SPEFSCR);
216 giveup_spe(tsk); 220 giveup_spe(tsk);
217 } 221 }
218 preempt_enable(); 222 preempt_enable();
diff --git a/arch/powerpc/kernel/setup-common.c b/arch/powerpc/kernel/setup-common.c
index 79fca2651b65..22051ef04bd9 100644
--- a/arch/powerpc/kernel/setup-common.c
+++ b/arch/powerpc/kernel/setup-common.c
@@ -375,6 +375,9 @@ void __init check_for_initrd(void)
375 375
376int threads_per_core, threads_shift; 376int threads_per_core, threads_shift;
377cpumask_t threads_core_mask; 377cpumask_t threads_core_mask;
378EXPORT_SYMBOL_GPL(threads_per_core);
379EXPORT_SYMBOL_GPL(threads_shift);
380EXPORT_SYMBOL_GPL(threads_core_mask);
378 381
379static void __init cpu_init_thread_core_maps(int tpc) 382static void __init cpu_init_thread_core_maps(int tpc)
380{ 383{
diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c
index a88bf2713d41..532054f24ecb 100644
--- a/arch/powerpc/kernel/setup_64.c
+++ b/arch/powerpc/kernel/setup_64.c
@@ -63,6 +63,7 @@
63#include <asm/kexec.h> 63#include <asm/kexec.h>
64#include <asm/mmu_context.h> 64#include <asm/mmu_context.h>
65#include <asm/code-patching.h> 65#include <asm/code-patching.h>
66#include <asm/kvm_ppc.h>
66 67
67#include "setup.h" 68#include "setup.h"
68 69
@@ -580,6 +581,8 @@ void __init setup_arch(char **cmdline_p)
580 /* Initialize the MMU context management stuff */ 581 /* Initialize the MMU context management stuff */
581 mmu_context_init(); 582 mmu_context_init();
582 583
584 kvm_rma_init();
585
583 ppc64_boot_msg(0x15, "Setup Done"); 586 ppc64_boot_msg(0x15, "Setup Done");
584} 587}
585 588
diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c
index 8ebc6700b98d..09a85a9045d6 100644
--- a/arch/powerpc/kernel/smp.c
+++ b/arch/powerpc/kernel/smp.c
@@ -243,6 +243,7 @@ void smp_send_reschedule(int cpu)
243 if (likely(smp_ops)) 243 if (likely(smp_ops))
244 smp_ops->message_pass(cpu, PPC_MSG_RESCHEDULE); 244 smp_ops->message_pass(cpu, PPC_MSG_RESCHEDULE);
245} 245}
246EXPORT_SYMBOL_GPL(smp_send_reschedule);
246 247
247void arch_send_call_function_single_ipi(int cpu) 248void arch_send_call_function_single_ipi(int cpu)
248{ 249{
diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c
index 1a0141426cda..f19d9777d3c1 100644
--- a/arch/powerpc/kernel/traps.c
+++ b/arch/powerpc/kernel/traps.c
@@ -1387,10 +1387,7 @@ void SPEFloatingPointException(struct pt_regs *regs)
1387 int code = 0; 1387 int code = 0;
1388 int err; 1388 int err;
1389 1389
1390 preempt_disable(); 1390 flush_spe_to_thread(current);
1391 if (regs->msr & MSR_SPE)
1392 giveup_spe(current);
1393 preempt_enable();
1394 1391
1395 spefscr = current->thread.spefscr; 1392 spefscr = current->thread.spefscr;
1396 fpexc_mode = current->thread.fpexc_mode; 1393 fpexc_mode = current->thread.fpexc_mode;
diff --git a/arch/powerpc/kvm/44x_tlb.c b/arch/powerpc/kvm/44x_tlb.c
index 5f3cff83e089..33aa715dab28 100644
--- a/arch/powerpc/kvm/44x_tlb.c
+++ b/arch/powerpc/kvm/44x_tlb.c
@@ -387,8 +387,10 @@ static void kvmppc_44x_invalidate(struct kvm_vcpu *vcpu,
387 } 387 }
388} 388}
389 389
390void kvmppc_mmu_priv_switch(struct kvm_vcpu *vcpu, int usermode) 390void kvmppc_mmu_msr_notify(struct kvm_vcpu *vcpu, u32 old_msr)
391{ 391{
392 int usermode = vcpu->arch.shared->msr & MSR_PR;
393
392 vcpu->arch.shadow_pid = !usermode; 394 vcpu->arch.shadow_pid = !usermode;
393} 395}
394 396
diff --git a/arch/powerpc/kvm/Kconfig b/arch/powerpc/kvm/Kconfig
index 105b6918b23e..78133deb4b64 100644
--- a/arch/powerpc/kvm/Kconfig
+++ b/arch/powerpc/kvm/Kconfig
@@ -20,7 +20,6 @@ config KVM
20 bool 20 bool
21 select PREEMPT_NOTIFIERS 21 select PREEMPT_NOTIFIERS
22 select ANON_INODES 22 select ANON_INODES
23 select KVM_MMIO
24 23
25config KVM_BOOK3S_HANDLER 24config KVM_BOOK3S_HANDLER
26 bool 25 bool
@@ -28,16 +27,22 @@ config KVM_BOOK3S_HANDLER
28config KVM_BOOK3S_32_HANDLER 27config KVM_BOOK3S_32_HANDLER
29 bool 28 bool
30 select KVM_BOOK3S_HANDLER 29 select KVM_BOOK3S_HANDLER
30 select KVM_MMIO
31 31
32config KVM_BOOK3S_64_HANDLER 32config KVM_BOOK3S_64_HANDLER
33 bool 33 bool
34 select KVM_BOOK3S_HANDLER 34 select KVM_BOOK3S_HANDLER
35 35
36config KVM_BOOK3S_PR
37 bool
38 select KVM_MMIO
39
36config KVM_BOOK3S_32 40config KVM_BOOK3S_32
37 tristate "KVM support for PowerPC book3s_32 processors" 41 tristate "KVM support for PowerPC book3s_32 processors"
38 depends on EXPERIMENTAL && PPC_BOOK3S_32 && !SMP && !PTE_64BIT 42 depends on EXPERIMENTAL && PPC_BOOK3S_32 && !SMP && !PTE_64BIT
39 select KVM 43 select KVM
40 select KVM_BOOK3S_32_HANDLER 44 select KVM_BOOK3S_32_HANDLER
45 select KVM_BOOK3S_PR
41 ---help--- 46 ---help---
42 Support running unmodified book3s_32 guest kernels 47 Support running unmodified book3s_32 guest kernels
43 in virtual machines on book3s_32 host processors. 48 in virtual machines on book3s_32 host processors.
@@ -50,8 +55,8 @@ config KVM_BOOK3S_32
50config KVM_BOOK3S_64 55config KVM_BOOK3S_64
51 tristate "KVM support for PowerPC book3s_64 processors" 56 tristate "KVM support for PowerPC book3s_64 processors"
52 depends on EXPERIMENTAL && PPC_BOOK3S_64 57 depends on EXPERIMENTAL && PPC_BOOK3S_64
53 select KVM
54 select KVM_BOOK3S_64_HANDLER 58 select KVM_BOOK3S_64_HANDLER
59 select KVM
55 ---help--- 60 ---help---
56 Support running unmodified book3s_64 and book3s_32 guest kernels 61 Support running unmodified book3s_64 and book3s_32 guest kernels
57 in virtual machines on book3s_64 host processors. 62 in virtual machines on book3s_64 host processors.
@@ -61,10 +66,34 @@ config KVM_BOOK3S_64
61 66
62 If unsure, say N. 67 If unsure, say N.
63 68
69config KVM_BOOK3S_64_HV
70 bool "KVM support for POWER7 and PPC970 using hypervisor mode in host"
71 depends on KVM_BOOK3S_64
72 ---help---
73 Support running unmodified book3s_64 guest kernels in
74 virtual machines on POWER7 and PPC970 processors that have
75 hypervisor mode available to the host.
76
77 If you say Y here, KVM will use the hardware virtualization
78 facilities of POWER7 (and later) processors, meaning that
79 guest operating systems will run at full hardware speed
80 using supervisor and user modes. However, this also means
81 that KVM is not usable under PowerVM (pHyp), is only usable
82 on POWER7 (or later) processors and PPC970-family processors,
83 and cannot emulate a different processor from the host processor.
84
85 If unsure, say N.
86
87config KVM_BOOK3S_64_PR
88 def_bool y
89 depends on KVM_BOOK3S_64 && !KVM_BOOK3S_64_HV
90 select KVM_BOOK3S_PR
91
64config KVM_440 92config KVM_440
65 bool "KVM support for PowerPC 440 processors" 93 bool "KVM support for PowerPC 440 processors"
66 depends on EXPERIMENTAL && 44x 94 depends on EXPERIMENTAL && 44x
67 select KVM 95 select KVM
96 select KVM_MMIO
68 ---help--- 97 ---help---
69 Support running unmodified 440 guest kernels in virtual machines on 98 Support running unmodified 440 guest kernels in virtual machines on
70 440 host processors. 99 440 host processors.
@@ -89,6 +118,7 @@ config KVM_E500
89 bool "KVM support for PowerPC E500 processors" 118 bool "KVM support for PowerPC E500 processors"
90 depends on EXPERIMENTAL && E500 119 depends on EXPERIMENTAL && E500
91 select KVM 120 select KVM
121 select KVM_MMIO
92 ---help--- 122 ---help---
93 Support running unmodified E500 guest kernels in virtual machines on 123 Support running unmodified E500 guest kernels in virtual machines on
94 E500 host processors. 124 E500 host processors.
diff --git a/arch/powerpc/kvm/Makefile b/arch/powerpc/kvm/Makefile
index 4d6863823f69..08428e2c188d 100644
--- a/arch/powerpc/kvm/Makefile
+++ b/arch/powerpc/kvm/Makefile
@@ -38,24 +38,42 @@ kvm-e500-objs := \
38 e500_emulate.o 38 e500_emulate.o
39kvm-objs-$(CONFIG_KVM_E500) := $(kvm-e500-objs) 39kvm-objs-$(CONFIG_KVM_E500) := $(kvm-e500-objs)
40 40
41kvm-book3s_64-objs := \ 41kvm-book3s_64-objs-$(CONFIG_KVM_BOOK3S_64_PR) := \
42 $(common-objs-y) \ 42 ../../../virt/kvm/coalesced_mmio.o \
43 fpu.o \ 43 fpu.o \
44 book3s_paired_singles.o \ 44 book3s_paired_singles.o \
45 book3s.o \ 45 book3s_pr.o \
46 book3s_emulate.o \ 46 book3s_emulate.o \
47 book3s_interrupts.o \ 47 book3s_interrupts.o \
48 book3s_mmu_hpte.o \ 48 book3s_mmu_hpte.o \
49 book3s_64_mmu_host.o \ 49 book3s_64_mmu_host.o \
50 book3s_64_mmu.o \ 50 book3s_64_mmu.o \
51 book3s_32_mmu.o 51 book3s_32_mmu.o
52kvm-objs-$(CONFIG_KVM_BOOK3S_64) := $(kvm-book3s_64-objs) 52
53kvm-book3s_64-objs-$(CONFIG_KVM_BOOK3S_64_HV) := \
54 book3s_hv.o \
55 book3s_hv_interrupts.o \
56 book3s_64_mmu_hv.o
57kvm-book3s_64-builtin-objs-$(CONFIG_KVM_BOOK3S_64_HV) := \
58 book3s_hv_rm_mmu.o \
59 book3s_64_vio_hv.o \
60 book3s_hv_builtin.o
61
62kvm-book3s_64-module-objs := \
63 ../../../virt/kvm/kvm_main.o \
64 powerpc.o \
65 emulate.o \
66 book3s.o \
67 $(kvm-book3s_64-objs-y)
68
69kvm-objs-$(CONFIG_KVM_BOOK3S_64) := $(kvm-book3s_64-module-objs)
53 70
54kvm-book3s_32-objs := \ 71kvm-book3s_32-objs := \
55 $(common-objs-y) \ 72 $(common-objs-y) \
56 fpu.o \ 73 fpu.o \
57 book3s_paired_singles.o \ 74 book3s_paired_singles.o \
58 book3s.o \ 75 book3s.o \
76 book3s_pr.o \
59 book3s_emulate.o \ 77 book3s_emulate.o \
60 book3s_interrupts.o \ 78 book3s_interrupts.o \
61 book3s_mmu_hpte.o \ 79 book3s_mmu_hpte.o \
@@ -70,3 +88,4 @@ obj-$(CONFIG_KVM_E500) += kvm.o
70obj-$(CONFIG_KVM_BOOK3S_64) += kvm.o 88obj-$(CONFIG_KVM_BOOK3S_64) += kvm.o
71obj-$(CONFIG_KVM_BOOK3S_32) += kvm.o 89obj-$(CONFIG_KVM_BOOK3S_32) += kvm.o
72 90
91obj-y += $(kvm-book3s_64-builtin-objs-y)
diff --git a/arch/powerpc/kvm/book3s.c b/arch/powerpc/kvm/book3s.c
index 0f95b5cce033..f68a34d16035 100644
--- a/arch/powerpc/kvm/book3s.c
+++ b/arch/powerpc/kvm/book3s.c
@@ -17,7 +17,6 @@
17#include <linux/kvm_host.h> 17#include <linux/kvm_host.h>
18#include <linux/err.h> 18#include <linux/err.h>
19#include <linux/slab.h> 19#include <linux/slab.h>
20#include "trace.h"
21 20
22#include <asm/reg.h> 21#include <asm/reg.h>
23#include <asm/cputable.h> 22#include <asm/cputable.h>
@@ -28,25 +27,17 @@
28#include <asm/kvm_ppc.h> 27#include <asm/kvm_ppc.h>
29#include <asm/kvm_book3s.h> 28#include <asm/kvm_book3s.h>
30#include <asm/mmu_context.h> 29#include <asm/mmu_context.h>
30#include <asm/page.h>
31#include <linux/gfp.h> 31#include <linux/gfp.h>
32#include <linux/sched.h> 32#include <linux/sched.h>
33#include <linux/vmalloc.h> 33#include <linux/vmalloc.h>
34#include <linux/highmem.h> 34#include <linux/highmem.h>
35 35
36#include "trace.h"
37
36#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU 38#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
37 39
38/* #define EXIT_DEBUG */ 40/* #define EXIT_DEBUG */
39/* #define DEBUG_EXT */
40
41static int kvmppc_handle_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr,
42 ulong msr);
43
44/* Some compatibility defines */
45#ifdef CONFIG_PPC_BOOK3S_32
46#define MSR_USER32 MSR_USER
47#define MSR_USER64 MSR_USER
48#define HW_PAGE_SIZE PAGE_SIZE
49#endif
50 41
51struct kvm_stats_debugfs_item debugfs_entries[] = { 42struct kvm_stats_debugfs_item debugfs_entries[] = {
52 { "exits", VCPU_STAT(sum_exits) }, 43 { "exits", VCPU_STAT(sum_exits) },
@@ -77,100 +68,11 @@ void kvmppc_core_load_guest_debugstate(struct kvm_vcpu *vcpu)
77{ 68{
78} 69}
79 70
80void kvmppc_core_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
81{
82#ifdef CONFIG_PPC_BOOK3S_64
83 memcpy(to_svcpu(vcpu)->slb, to_book3s(vcpu)->slb_shadow, sizeof(to_svcpu(vcpu)->slb));
84 memcpy(&get_paca()->shadow_vcpu, to_book3s(vcpu)->shadow_vcpu,
85 sizeof(get_paca()->shadow_vcpu));
86 to_svcpu(vcpu)->slb_max = to_book3s(vcpu)->slb_shadow_max;
87#endif
88
89#ifdef CONFIG_PPC_BOOK3S_32
90 current->thread.kvm_shadow_vcpu = to_book3s(vcpu)->shadow_vcpu;
91#endif
92}
93
94void kvmppc_core_vcpu_put(struct kvm_vcpu *vcpu)
95{
96#ifdef CONFIG_PPC_BOOK3S_64
97 memcpy(to_book3s(vcpu)->slb_shadow, to_svcpu(vcpu)->slb, sizeof(to_svcpu(vcpu)->slb));
98 memcpy(to_book3s(vcpu)->shadow_vcpu, &get_paca()->shadow_vcpu,
99 sizeof(get_paca()->shadow_vcpu));
100 to_book3s(vcpu)->slb_shadow_max = to_svcpu(vcpu)->slb_max;
101#endif
102
103 kvmppc_giveup_ext(vcpu, MSR_FP);
104 kvmppc_giveup_ext(vcpu, MSR_VEC);
105 kvmppc_giveup_ext(vcpu, MSR_VSX);
106}
107
108static void kvmppc_recalc_shadow_msr(struct kvm_vcpu *vcpu)
109{
110 ulong smsr = vcpu->arch.shared->msr;
111
112 /* Guest MSR values */
113 smsr &= MSR_FE0 | MSR_FE1 | MSR_SF | MSR_SE | MSR_BE | MSR_DE;
114 /* Process MSR values */
115 smsr |= MSR_ME | MSR_RI | MSR_IR | MSR_DR | MSR_PR | MSR_EE;
116 /* External providers the guest reserved */
117 smsr |= (vcpu->arch.shared->msr & vcpu->arch.guest_owned_ext);
118 /* 64-bit Process MSR values */
119#ifdef CONFIG_PPC_BOOK3S_64
120 smsr |= MSR_ISF | MSR_HV;
121#endif
122 vcpu->arch.shadow_msr = smsr;
123}
124
125void kvmppc_set_msr(struct kvm_vcpu *vcpu, u64 msr)
126{
127 ulong old_msr = vcpu->arch.shared->msr;
128
129#ifdef EXIT_DEBUG
130 printk(KERN_INFO "KVM: Set MSR to 0x%llx\n", msr);
131#endif
132
133 msr &= to_book3s(vcpu)->msr_mask;
134 vcpu->arch.shared->msr = msr;
135 kvmppc_recalc_shadow_msr(vcpu);
136
137 if (msr & MSR_POW) {
138 if (!vcpu->arch.pending_exceptions) {
139 kvm_vcpu_block(vcpu);
140 vcpu->stat.halt_wakeup++;
141
142 /* Unset POW bit after we woke up */
143 msr &= ~MSR_POW;
144 vcpu->arch.shared->msr = msr;
145 }
146 }
147
148 if ((vcpu->arch.shared->msr & (MSR_PR|MSR_IR|MSR_DR)) !=
149 (old_msr & (MSR_PR|MSR_IR|MSR_DR))) {
150 kvmppc_mmu_flush_segments(vcpu);
151 kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu));
152
153 /* Preload magic page segment when in kernel mode */
154 if (!(msr & MSR_PR) && vcpu->arch.magic_page_pa) {
155 struct kvm_vcpu_arch *a = &vcpu->arch;
156
157 if (msr & MSR_DR)
158 kvmppc_mmu_map_segment(vcpu, a->magic_page_ea);
159 else
160 kvmppc_mmu_map_segment(vcpu, a->magic_page_pa);
161 }
162 }
163
164 /* Preload FPU if it's enabled */
165 if (vcpu->arch.shared->msr & MSR_FP)
166 kvmppc_handle_ext(vcpu, BOOK3S_INTERRUPT_FP_UNAVAIL, MSR_FP);
167}
168
169void kvmppc_inject_interrupt(struct kvm_vcpu *vcpu, int vec, u64 flags) 71void kvmppc_inject_interrupt(struct kvm_vcpu *vcpu, int vec, u64 flags)
170{ 72{
171 vcpu->arch.shared->srr0 = kvmppc_get_pc(vcpu); 73 vcpu->arch.shared->srr0 = kvmppc_get_pc(vcpu);
172 vcpu->arch.shared->srr1 = vcpu->arch.shared->msr | flags; 74 vcpu->arch.shared->srr1 = vcpu->arch.shared->msr | flags;
173 kvmppc_set_pc(vcpu, to_book3s(vcpu)->hior + vec); 75 kvmppc_set_pc(vcpu, kvmppc_interrupt_offset(vcpu) + vec);
174 vcpu->arch.mmu.reset_msr(vcpu); 76 vcpu->arch.mmu.reset_msr(vcpu);
175} 77}
176 78
@@ -204,11 +106,13 @@ static int kvmppc_book3s_vec2irqprio(unsigned int vec)
204static void kvmppc_book3s_dequeue_irqprio(struct kvm_vcpu *vcpu, 106static void kvmppc_book3s_dequeue_irqprio(struct kvm_vcpu *vcpu,
205 unsigned int vec) 107 unsigned int vec)
206{ 108{
109 unsigned long old_pending = vcpu->arch.pending_exceptions;
110
207 clear_bit(kvmppc_book3s_vec2irqprio(vec), 111 clear_bit(kvmppc_book3s_vec2irqprio(vec),
208 &vcpu->arch.pending_exceptions); 112 &vcpu->arch.pending_exceptions);
209 113
210 if (!vcpu->arch.pending_exceptions) 114 kvmppc_update_int_pending(vcpu, vcpu->arch.pending_exceptions,
211 vcpu->arch.shared->int_pending = 0; 115 old_pending);
212} 116}
213 117
214void kvmppc_book3s_queue_irqprio(struct kvm_vcpu *vcpu, unsigned int vec) 118void kvmppc_book3s_queue_irqprio(struct kvm_vcpu *vcpu, unsigned int vec)
@@ -225,8 +129,8 @@ void kvmppc_book3s_queue_irqprio(struct kvm_vcpu *vcpu, unsigned int vec)
225 129
226void kvmppc_core_queue_program(struct kvm_vcpu *vcpu, ulong flags) 130void kvmppc_core_queue_program(struct kvm_vcpu *vcpu, ulong flags)
227{ 131{
228 to_book3s(vcpu)->prog_flags = flags; 132 /* might as well deliver this straight away */
229 kvmppc_book3s_queue_irqprio(vcpu, BOOK3S_INTERRUPT_PROGRAM); 133 kvmppc_inject_interrupt(vcpu, BOOK3S_INTERRUPT_PROGRAM, flags);
230} 134}
231 135
232void kvmppc_core_queue_dec(struct kvm_vcpu *vcpu) 136void kvmppc_core_queue_dec(struct kvm_vcpu *vcpu)
@@ -266,21 +170,7 @@ int kvmppc_book3s_irqprio_deliver(struct kvm_vcpu *vcpu, unsigned int priority)
266{ 170{
267 int deliver = 1; 171 int deliver = 1;
268 int vec = 0; 172 int vec = 0;
269 ulong flags = 0ULL; 173 bool crit = kvmppc_critical_section(vcpu);
270 ulong crit_raw = vcpu->arch.shared->critical;
271 ulong crit_r1 = kvmppc_get_gpr(vcpu, 1);
272 bool crit;
273
274 /* Truncate crit indicators in 32 bit mode */
275 if (!(vcpu->arch.shared->msr & MSR_SF)) {
276 crit_raw &= 0xffffffff;
277 crit_r1 &= 0xffffffff;
278 }
279
280 /* Critical section when crit == r1 */
281 crit = (crit_raw == crit_r1);
282 /* ... and we're in supervisor mode */
283 crit = crit && !(vcpu->arch.shared->msr & MSR_PR);
284 174
285 switch (priority) { 175 switch (priority) {
286 case BOOK3S_IRQPRIO_DECREMENTER: 176 case BOOK3S_IRQPRIO_DECREMENTER:
@@ -315,7 +205,6 @@ int kvmppc_book3s_irqprio_deliver(struct kvm_vcpu *vcpu, unsigned int priority)
315 break; 205 break;
316 case BOOK3S_IRQPRIO_PROGRAM: 206 case BOOK3S_IRQPRIO_PROGRAM:
317 vec = BOOK3S_INTERRUPT_PROGRAM; 207 vec = BOOK3S_INTERRUPT_PROGRAM;
318 flags = to_book3s(vcpu)->prog_flags;
319 break; 208 break;
320 case BOOK3S_IRQPRIO_VSX: 209 case BOOK3S_IRQPRIO_VSX:
321 vec = BOOK3S_INTERRUPT_VSX; 210 vec = BOOK3S_INTERRUPT_VSX;
@@ -346,7 +235,7 @@ int kvmppc_book3s_irqprio_deliver(struct kvm_vcpu *vcpu, unsigned int priority)
346#endif 235#endif
347 236
348 if (deliver) 237 if (deliver)
349 kvmppc_inject_interrupt(vcpu, vec, flags); 238 kvmppc_inject_interrupt(vcpu, vec, 0);
350 239
351 return deliver; 240 return deliver;
352} 241}
@@ -392,64 +281,7 @@ void kvmppc_core_deliver_interrupts(struct kvm_vcpu *vcpu)
392 } 281 }
393 282
394 /* Tell the guest about our interrupt status */ 283 /* Tell the guest about our interrupt status */
395 if (*pending) 284 kvmppc_update_int_pending(vcpu, *pending, old_pending);
396 vcpu->arch.shared->int_pending = 1;
397 else if (old_pending)
398 vcpu->arch.shared->int_pending = 0;
399}
400
401void kvmppc_set_pvr(struct kvm_vcpu *vcpu, u32 pvr)
402{
403 u32 host_pvr;
404
405 vcpu->arch.hflags &= ~BOOK3S_HFLAG_SLB;
406 vcpu->arch.pvr = pvr;
407#ifdef CONFIG_PPC_BOOK3S_64
408 if ((pvr >= 0x330000) && (pvr < 0x70330000)) {
409 kvmppc_mmu_book3s_64_init(vcpu);
410 to_book3s(vcpu)->hior = 0xfff00000;
411 to_book3s(vcpu)->msr_mask = 0xffffffffffffffffULL;
412 } else
413#endif
414 {
415 kvmppc_mmu_book3s_32_init(vcpu);
416 to_book3s(vcpu)->hior = 0;
417 to_book3s(vcpu)->msr_mask = 0xffffffffULL;
418 }
419
420 /* If we are in hypervisor level on 970, we can tell the CPU to
421 * treat DCBZ as 32 bytes store */
422 vcpu->arch.hflags &= ~BOOK3S_HFLAG_DCBZ32;
423 if (vcpu->arch.mmu.is_dcbz32(vcpu) && (mfmsr() & MSR_HV) &&
424 !strcmp(cur_cpu_spec->platform, "ppc970"))
425 vcpu->arch.hflags |= BOOK3S_HFLAG_DCBZ32;
426
427 /* Cell performs badly if MSR_FEx are set. So let's hope nobody
428 really needs them in a VM on Cell and force disable them. */
429 if (!strcmp(cur_cpu_spec->platform, "ppc-cell-be"))
430 to_book3s(vcpu)->msr_mask &= ~(MSR_FE0 | MSR_FE1);
431
432#ifdef CONFIG_PPC_BOOK3S_32
433 /* 32 bit Book3S always has 32 byte dcbz */
434 vcpu->arch.hflags |= BOOK3S_HFLAG_DCBZ32;
435#endif
436
437 /* On some CPUs we can execute paired single operations natively */
438 asm ( "mfpvr %0" : "=r"(host_pvr));
439 switch (host_pvr) {
440 case 0x00080200: /* lonestar 2.0 */
441 case 0x00088202: /* lonestar 2.2 */
442 case 0x70000100: /* gekko 1.0 */
443 case 0x00080100: /* gekko 2.0 */
444 case 0x00083203: /* gekko 2.3a */
445 case 0x00083213: /* gekko 2.3b */
446 case 0x00083204: /* gekko 2.4 */
447 case 0x00083214: /* gekko 2.4e (8SE) - retail HW2 */
448 case 0x00087200: /* broadway */
449 vcpu->arch.hflags |= BOOK3S_HFLAG_NATIVE_PS;
450 /* Enable HID2.PSE - in case we need it later */
451 mtspr(SPRN_HID2_GEKKO, mfspr(SPRN_HID2_GEKKO) | (1 << 29));
452 }
453} 285}
454 286
455pfn_t kvmppc_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn) 287pfn_t kvmppc_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn)
@@ -471,44 +303,6 @@ pfn_t kvmppc_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn)
471 return gfn_to_pfn(vcpu->kvm, gfn); 303 return gfn_to_pfn(vcpu->kvm, gfn);
472} 304}
473 305
474/* Book3s_32 CPUs always have 32 bytes cache line size, which Linux assumes. To
475 * make Book3s_32 Linux work on Book3s_64, we have to make sure we trap dcbz to
476 * emulate 32 bytes dcbz length.
477 *
478 * The Book3s_64 inventors also realized this case and implemented a special bit
479 * in the HID5 register, which is a hypervisor ressource. Thus we can't use it.
480 *
481 * My approach here is to patch the dcbz instruction on executing pages.
482 */
483static void kvmppc_patch_dcbz(struct kvm_vcpu *vcpu, struct kvmppc_pte *pte)
484{
485 struct page *hpage;
486 u64 hpage_offset;
487 u32 *page;
488 int i;
489
490 hpage = gfn_to_page(vcpu->kvm, pte->raddr >> PAGE_SHIFT);
491 if (is_error_page(hpage)) {
492 kvm_release_page_clean(hpage);
493 return;
494 }
495
496 hpage_offset = pte->raddr & ~PAGE_MASK;
497 hpage_offset &= ~0xFFFULL;
498 hpage_offset /= 4;
499
500 get_page(hpage);
501 page = kmap_atomic(hpage, KM_USER0);
502
503 /* patch dcbz into reserved instruction, so we trap */
504 for (i=hpage_offset; i < hpage_offset + (HW_PAGE_SIZE / 4); i++)
505 if ((page[i] & 0xff0007ff) == INS_DCBZ)
506 page[i] &= 0xfffffff7;
507
508 kunmap_atomic(page, KM_USER0);
509 put_page(hpage);
510}
511
512static int kvmppc_xlate(struct kvm_vcpu *vcpu, ulong eaddr, bool data, 306static int kvmppc_xlate(struct kvm_vcpu *vcpu, ulong eaddr, bool data,
513 struct kvmppc_pte *pte) 307 struct kvmppc_pte *pte)
514{ 308{
@@ -606,519 +400,6 @@ mmio:
606 return EMULATE_DO_MMIO; 400 return EMULATE_DO_MMIO;
607} 401}
608 402
609static int kvmppc_visible_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
610{
611 ulong mp_pa = vcpu->arch.magic_page_pa;
612
613 if (unlikely(mp_pa) &&
614 unlikely((mp_pa & KVM_PAM) >> PAGE_SHIFT == gfn)) {
615 return 1;
616 }
617
618 return kvm_is_visible_gfn(vcpu->kvm, gfn);
619}
620
621int kvmppc_handle_pagefault(struct kvm_run *run, struct kvm_vcpu *vcpu,
622 ulong eaddr, int vec)
623{
624 bool data = (vec == BOOK3S_INTERRUPT_DATA_STORAGE);
625 int r = RESUME_GUEST;
626 int relocated;
627 int page_found = 0;
628 struct kvmppc_pte pte;
629 bool is_mmio = false;
630 bool dr = (vcpu->arch.shared->msr & MSR_DR) ? true : false;
631 bool ir = (vcpu->arch.shared->msr & MSR_IR) ? true : false;
632 u64 vsid;
633
634 relocated = data ? dr : ir;
635
636 /* Resolve real address if translation turned on */
637 if (relocated) {
638 page_found = vcpu->arch.mmu.xlate(vcpu, eaddr, &pte, data);
639 } else {
640 pte.may_execute = true;
641 pte.may_read = true;
642 pte.may_write = true;
643 pte.raddr = eaddr & KVM_PAM;
644 pte.eaddr = eaddr;
645 pte.vpage = eaddr >> 12;
646 }
647
648 switch (vcpu->arch.shared->msr & (MSR_DR|MSR_IR)) {
649 case 0:
650 pte.vpage |= ((u64)VSID_REAL << (SID_SHIFT - 12));
651 break;
652 case MSR_DR:
653 case MSR_IR:
654 vcpu->arch.mmu.esid_to_vsid(vcpu, eaddr >> SID_SHIFT, &vsid);
655
656 if ((vcpu->arch.shared->msr & (MSR_DR|MSR_IR)) == MSR_DR)
657 pte.vpage |= ((u64)VSID_REAL_DR << (SID_SHIFT - 12));
658 else
659 pte.vpage |= ((u64)VSID_REAL_IR << (SID_SHIFT - 12));
660 pte.vpage |= vsid;
661
662 if (vsid == -1)
663 page_found = -EINVAL;
664 break;
665 }
666
667 if (vcpu->arch.mmu.is_dcbz32(vcpu) &&
668 (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32))) {
669 /*
670 * If we do the dcbz hack, we have to NX on every execution,
671 * so we can patch the executing code. This renders our guest
672 * NX-less.
673 */
674 pte.may_execute = !data;
675 }
676
677 if (page_found == -ENOENT) {
678 /* Page not found in guest PTE entries */
679 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu);
680 vcpu->arch.shared->dsisr = to_svcpu(vcpu)->fault_dsisr;
681 vcpu->arch.shared->msr |=
682 (to_svcpu(vcpu)->shadow_srr1 & 0x00000000f8000000ULL);
683 kvmppc_book3s_queue_irqprio(vcpu, vec);
684 } else if (page_found == -EPERM) {
685 /* Storage protection */
686 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu);
687 vcpu->arch.shared->dsisr =
688 to_svcpu(vcpu)->fault_dsisr & ~DSISR_NOHPTE;
689 vcpu->arch.shared->dsisr |= DSISR_PROTFAULT;
690 vcpu->arch.shared->msr |=
691 (to_svcpu(vcpu)->shadow_srr1 & 0x00000000f8000000ULL);
692 kvmppc_book3s_queue_irqprio(vcpu, vec);
693 } else if (page_found == -EINVAL) {
694 /* Page not found in guest SLB */
695 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu);
696 kvmppc_book3s_queue_irqprio(vcpu, vec + 0x80);
697 } else if (!is_mmio &&
698 kvmppc_visible_gfn(vcpu, pte.raddr >> PAGE_SHIFT)) {
699 /* The guest's PTE is not mapped yet. Map on the host */
700 kvmppc_mmu_map_page(vcpu, &pte);
701 if (data)
702 vcpu->stat.sp_storage++;
703 else if (vcpu->arch.mmu.is_dcbz32(vcpu) &&
704 (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32)))
705 kvmppc_patch_dcbz(vcpu, &pte);
706 } else {
707 /* MMIO */
708 vcpu->stat.mmio_exits++;
709 vcpu->arch.paddr_accessed = pte.raddr;
710 r = kvmppc_emulate_mmio(run, vcpu);
711 if ( r == RESUME_HOST_NV )
712 r = RESUME_HOST;
713 }
714
715 return r;
716}
717
718static inline int get_fpr_index(int i)
719{
720#ifdef CONFIG_VSX
721 i *= 2;
722#endif
723 return i;
724}
725
726/* Give up external provider (FPU, Altivec, VSX) */
727void kvmppc_giveup_ext(struct kvm_vcpu *vcpu, ulong msr)
728{
729 struct thread_struct *t = &current->thread;
730 u64 *vcpu_fpr = vcpu->arch.fpr;
731#ifdef CONFIG_VSX
732 u64 *vcpu_vsx = vcpu->arch.vsr;
733#endif
734 u64 *thread_fpr = (u64*)t->fpr;
735 int i;
736
737 if (!(vcpu->arch.guest_owned_ext & msr))
738 return;
739
740#ifdef DEBUG_EXT
741 printk(KERN_INFO "Giving up ext 0x%lx\n", msr);
742#endif
743
744 switch (msr) {
745 case MSR_FP:
746 giveup_fpu(current);
747 for (i = 0; i < ARRAY_SIZE(vcpu->arch.fpr); i++)
748 vcpu_fpr[i] = thread_fpr[get_fpr_index(i)];
749
750 vcpu->arch.fpscr = t->fpscr.val;
751 break;
752 case MSR_VEC:
753#ifdef CONFIG_ALTIVEC
754 giveup_altivec(current);
755 memcpy(vcpu->arch.vr, t->vr, sizeof(vcpu->arch.vr));
756 vcpu->arch.vscr = t->vscr;
757#endif
758 break;
759 case MSR_VSX:
760#ifdef CONFIG_VSX
761 __giveup_vsx(current);
762 for (i = 0; i < ARRAY_SIZE(vcpu->arch.vsr); i++)
763 vcpu_vsx[i] = thread_fpr[get_fpr_index(i) + 1];
764#endif
765 break;
766 default:
767 BUG();
768 }
769
770 vcpu->arch.guest_owned_ext &= ~msr;
771 current->thread.regs->msr &= ~msr;
772 kvmppc_recalc_shadow_msr(vcpu);
773}
774
775static int kvmppc_read_inst(struct kvm_vcpu *vcpu)
776{
777 ulong srr0 = kvmppc_get_pc(vcpu);
778 u32 last_inst = kvmppc_get_last_inst(vcpu);
779 int ret;
780
781 ret = kvmppc_ld(vcpu, &srr0, sizeof(u32), &last_inst, false);
782 if (ret == -ENOENT) {
783 ulong msr = vcpu->arch.shared->msr;
784
785 msr = kvmppc_set_field(msr, 33, 33, 1);
786 msr = kvmppc_set_field(msr, 34, 36, 0);
787 vcpu->arch.shared->msr = kvmppc_set_field(msr, 42, 47, 0);
788 kvmppc_book3s_queue_irqprio(vcpu, BOOK3S_INTERRUPT_INST_STORAGE);
789 return EMULATE_AGAIN;
790 }
791
792 return EMULATE_DONE;
793}
794
795static int kvmppc_check_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr)
796{
797
798 /* Need to do paired single emulation? */
799 if (!(vcpu->arch.hflags & BOOK3S_HFLAG_PAIRED_SINGLE))
800 return EMULATE_DONE;
801
802 /* Read out the instruction */
803 if (kvmppc_read_inst(vcpu) == EMULATE_DONE)
804 /* Need to emulate */
805 return EMULATE_FAIL;
806
807 return EMULATE_AGAIN;
808}
809
810/* Handle external providers (FPU, Altivec, VSX) */
811static int kvmppc_handle_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr,
812 ulong msr)
813{
814 struct thread_struct *t = &current->thread;
815 u64 *vcpu_fpr = vcpu->arch.fpr;
816#ifdef CONFIG_VSX
817 u64 *vcpu_vsx = vcpu->arch.vsr;
818#endif
819 u64 *thread_fpr = (u64*)t->fpr;
820 int i;
821
822 /* When we have paired singles, we emulate in software */
823 if (vcpu->arch.hflags & BOOK3S_HFLAG_PAIRED_SINGLE)
824 return RESUME_GUEST;
825
826 if (!(vcpu->arch.shared->msr & msr)) {
827 kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
828 return RESUME_GUEST;
829 }
830
831 /* We already own the ext */
832 if (vcpu->arch.guest_owned_ext & msr) {
833 return RESUME_GUEST;
834 }
835
836#ifdef DEBUG_EXT
837 printk(KERN_INFO "Loading up ext 0x%lx\n", msr);
838#endif
839
840 current->thread.regs->msr |= msr;
841
842 switch (msr) {
843 case MSR_FP:
844 for (i = 0; i < ARRAY_SIZE(vcpu->arch.fpr); i++)
845 thread_fpr[get_fpr_index(i)] = vcpu_fpr[i];
846
847 t->fpscr.val = vcpu->arch.fpscr;
848 t->fpexc_mode = 0;
849 kvmppc_load_up_fpu();
850 break;
851 case MSR_VEC:
852#ifdef CONFIG_ALTIVEC
853 memcpy(t->vr, vcpu->arch.vr, sizeof(vcpu->arch.vr));
854 t->vscr = vcpu->arch.vscr;
855 t->vrsave = -1;
856 kvmppc_load_up_altivec();
857#endif
858 break;
859 case MSR_VSX:
860#ifdef CONFIG_VSX
861 for (i = 0; i < ARRAY_SIZE(vcpu->arch.vsr); i++)
862 thread_fpr[get_fpr_index(i) + 1] = vcpu_vsx[i];
863 kvmppc_load_up_vsx();
864#endif
865 break;
866 default:
867 BUG();
868 }
869
870 vcpu->arch.guest_owned_ext |= msr;
871
872 kvmppc_recalc_shadow_msr(vcpu);
873
874 return RESUME_GUEST;
875}
876
877int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu,
878 unsigned int exit_nr)
879{
880 int r = RESUME_HOST;
881
882 vcpu->stat.sum_exits++;
883
884 run->exit_reason = KVM_EXIT_UNKNOWN;
885 run->ready_for_interrupt_injection = 1;
886
887 trace_kvm_book3s_exit(exit_nr, vcpu);
888 kvm_resched(vcpu);
889 switch (exit_nr) {
890 case BOOK3S_INTERRUPT_INST_STORAGE:
891 vcpu->stat.pf_instruc++;
892
893#ifdef CONFIG_PPC_BOOK3S_32
894 /* We set segments as unused segments when invalidating them. So
895 * treat the respective fault as segment fault. */
896 if (to_svcpu(vcpu)->sr[kvmppc_get_pc(vcpu) >> SID_SHIFT]
897 == SR_INVALID) {
898 kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu));
899 r = RESUME_GUEST;
900 break;
901 }
902#endif
903
904 /* only care about PTEG not found errors, but leave NX alone */
905 if (to_svcpu(vcpu)->shadow_srr1 & 0x40000000) {
906 r = kvmppc_handle_pagefault(run, vcpu, kvmppc_get_pc(vcpu), exit_nr);
907 vcpu->stat.sp_instruc++;
908 } else if (vcpu->arch.mmu.is_dcbz32(vcpu) &&
909 (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32))) {
910 /*
911 * XXX If we do the dcbz hack we use the NX bit to flush&patch the page,
912 * so we can't use the NX bit inside the guest. Let's cross our fingers,
913 * that no guest that needs the dcbz hack does NX.
914 */
915 kvmppc_mmu_pte_flush(vcpu, kvmppc_get_pc(vcpu), ~0xFFFUL);
916 r = RESUME_GUEST;
917 } else {
918 vcpu->arch.shared->msr |=
919 to_svcpu(vcpu)->shadow_srr1 & 0x58000000;
920 kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
921 r = RESUME_GUEST;
922 }
923 break;
924 case BOOK3S_INTERRUPT_DATA_STORAGE:
925 {
926 ulong dar = kvmppc_get_fault_dar(vcpu);
927 vcpu->stat.pf_storage++;
928
929#ifdef CONFIG_PPC_BOOK3S_32
930 /* We set segments as unused segments when invalidating them. So
931 * treat the respective fault as segment fault. */
932 if ((to_svcpu(vcpu)->sr[dar >> SID_SHIFT]) == SR_INVALID) {
933 kvmppc_mmu_map_segment(vcpu, dar);
934 r = RESUME_GUEST;
935 break;
936 }
937#endif
938
939 /* The only case we need to handle is missing shadow PTEs */
940 if (to_svcpu(vcpu)->fault_dsisr & DSISR_NOHPTE) {
941 r = kvmppc_handle_pagefault(run, vcpu, dar, exit_nr);
942 } else {
943 vcpu->arch.shared->dar = dar;
944 vcpu->arch.shared->dsisr = to_svcpu(vcpu)->fault_dsisr;
945 kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
946 r = RESUME_GUEST;
947 }
948 break;
949 }
950 case BOOK3S_INTERRUPT_DATA_SEGMENT:
951 if (kvmppc_mmu_map_segment(vcpu, kvmppc_get_fault_dar(vcpu)) < 0) {
952 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu);
953 kvmppc_book3s_queue_irqprio(vcpu,
954 BOOK3S_INTERRUPT_DATA_SEGMENT);
955 }
956 r = RESUME_GUEST;
957 break;
958 case BOOK3S_INTERRUPT_INST_SEGMENT:
959 if (kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu)) < 0) {
960 kvmppc_book3s_queue_irqprio(vcpu,
961 BOOK3S_INTERRUPT_INST_SEGMENT);
962 }
963 r = RESUME_GUEST;
964 break;
965 /* We're good on these - the host merely wanted to get our attention */
966 case BOOK3S_INTERRUPT_DECREMENTER:
967 vcpu->stat.dec_exits++;
968 r = RESUME_GUEST;
969 break;
970 case BOOK3S_INTERRUPT_EXTERNAL:
971 vcpu->stat.ext_intr_exits++;
972 r = RESUME_GUEST;
973 break;
974 case BOOK3S_INTERRUPT_PERFMON:
975 r = RESUME_GUEST;
976 break;
977 case BOOK3S_INTERRUPT_PROGRAM:
978 {
979 enum emulation_result er;
980 ulong flags;
981
982program_interrupt:
983 flags = to_svcpu(vcpu)->shadow_srr1 & 0x1f0000ull;
984
985 if (vcpu->arch.shared->msr & MSR_PR) {
986#ifdef EXIT_DEBUG
987 printk(KERN_INFO "Userspace triggered 0x700 exception at 0x%lx (0x%x)\n", kvmppc_get_pc(vcpu), kvmppc_get_last_inst(vcpu));
988#endif
989 if ((kvmppc_get_last_inst(vcpu) & 0xff0007ff) !=
990 (INS_DCBZ & 0xfffffff7)) {
991 kvmppc_core_queue_program(vcpu, flags);
992 r = RESUME_GUEST;
993 break;
994 }
995 }
996
997 vcpu->stat.emulated_inst_exits++;
998 er = kvmppc_emulate_instruction(run, vcpu);
999 switch (er) {
1000 case EMULATE_DONE:
1001 r = RESUME_GUEST_NV;
1002 break;
1003 case EMULATE_AGAIN:
1004 r = RESUME_GUEST;
1005 break;
1006 case EMULATE_FAIL:
1007 printk(KERN_CRIT "%s: emulation at %lx failed (%08x)\n",
1008 __func__, kvmppc_get_pc(vcpu), kvmppc_get_last_inst(vcpu));
1009 kvmppc_core_queue_program(vcpu, flags);
1010 r = RESUME_GUEST;
1011 break;
1012 case EMULATE_DO_MMIO:
1013 run->exit_reason = KVM_EXIT_MMIO;
1014 r = RESUME_HOST_NV;
1015 break;
1016 default:
1017 BUG();
1018 }
1019 break;
1020 }
1021 case BOOK3S_INTERRUPT_SYSCALL:
1022 if (vcpu->arch.osi_enabled &&
1023 (((u32)kvmppc_get_gpr(vcpu, 3)) == OSI_SC_MAGIC_R3) &&
1024 (((u32)kvmppc_get_gpr(vcpu, 4)) == OSI_SC_MAGIC_R4)) {
1025 /* MOL hypercalls */
1026 u64 *gprs = run->osi.gprs;
1027 int i;
1028
1029 run->exit_reason = KVM_EXIT_OSI;
1030 for (i = 0; i < 32; i++)
1031 gprs[i] = kvmppc_get_gpr(vcpu, i);
1032 vcpu->arch.osi_needed = 1;
1033 r = RESUME_HOST_NV;
1034 } else if (!(vcpu->arch.shared->msr & MSR_PR) &&
1035 (((u32)kvmppc_get_gpr(vcpu, 0)) == KVM_SC_MAGIC_R0)) {
1036 /* KVM PV hypercalls */
1037 kvmppc_set_gpr(vcpu, 3, kvmppc_kvm_pv(vcpu));
1038 r = RESUME_GUEST;
1039 } else {
1040 /* Guest syscalls */
1041 vcpu->stat.syscall_exits++;
1042 kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
1043 r = RESUME_GUEST;
1044 }
1045 break;
1046 case BOOK3S_INTERRUPT_FP_UNAVAIL:
1047 case BOOK3S_INTERRUPT_ALTIVEC:
1048 case BOOK3S_INTERRUPT_VSX:
1049 {
1050 int ext_msr = 0;
1051
1052 switch (exit_nr) {
1053 case BOOK3S_INTERRUPT_FP_UNAVAIL: ext_msr = MSR_FP; break;
1054 case BOOK3S_INTERRUPT_ALTIVEC: ext_msr = MSR_VEC; break;
1055 case BOOK3S_INTERRUPT_VSX: ext_msr = MSR_VSX; break;
1056 }
1057
1058 switch (kvmppc_check_ext(vcpu, exit_nr)) {
1059 case EMULATE_DONE:
1060 /* everything ok - let's enable the ext */
1061 r = kvmppc_handle_ext(vcpu, exit_nr, ext_msr);
1062 break;
1063 case EMULATE_FAIL:
1064 /* we need to emulate this instruction */
1065 goto program_interrupt;
1066 break;
1067 default:
1068 /* nothing to worry about - go again */
1069 break;
1070 }
1071 break;
1072 }
1073 case BOOK3S_INTERRUPT_ALIGNMENT:
1074 if (kvmppc_read_inst(vcpu) == EMULATE_DONE) {
1075 vcpu->arch.shared->dsisr = kvmppc_alignment_dsisr(vcpu,
1076 kvmppc_get_last_inst(vcpu));
1077 vcpu->arch.shared->dar = kvmppc_alignment_dar(vcpu,
1078 kvmppc_get_last_inst(vcpu));
1079 kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
1080 }
1081 r = RESUME_GUEST;
1082 break;
1083 case BOOK3S_INTERRUPT_MACHINE_CHECK:
1084 case BOOK3S_INTERRUPT_TRACE:
1085 kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
1086 r = RESUME_GUEST;
1087 break;
1088 default:
1089 /* Ugh - bork here! What did we get? */
1090 printk(KERN_EMERG "exit_nr=0x%x | pc=0x%lx | msr=0x%lx\n",
1091 exit_nr, kvmppc_get_pc(vcpu), to_svcpu(vcpu)->shadow_srr1);
1092 r = RESUME_HOST;
1093 BUG();
1094 break;
1095 }
1096
1097
1098 if (!(r & RESUME_HOST)) {
1099 /* To avoid clobbering exit_reason, only check for signals if
1100 * we aren't already exiting to userspace for some other
1101 * reason. */
1102 if (signal_pending(current)) {
1103#ifdef EXIT_DEBUG
1104 printk(KERN_EMERG "KVM: Going back to host\n");
1105#endif
1106 vcpu->stat.signal_exits++;
1107 run->exit_reason = KVM_EXIT_INTR;
1108 r = -EINTR;
1109 } else {
1110 /* In case an interrupt came in that was triggered
1111 * from userspace (like DEC), we need to check what
1112 * to inject now! */
1113 kvmppc_core_deliver_interrupts(vcpu);
1114 }
1115 }
1116
1117 trace_kvm_book3s_reenter(r, vcpu);
1118
1119 return r;
1120}
1121
1122int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) 403int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1123{ 404{
1124 return 0; 405 return 0;
@@ -1179,69 +460,6 @@ int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1179 return 0; 460 return 0;
1180} 461}
1181 462
1182int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1183 struct kvm_sregs *sregs)
1184{
1185 struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu);
1186 int i;
1187
1188 sregs->pvr = vcpu->arch.pvr;
1189
1190 sregs->u.s.sdr1 = to_book3s(vcpu)->sdr1;
1191 if (vcpu->arch.hflags & BOOK3S_HFLAG_SLB) {
1192 for (i = 0; i < 64; i++) {
1193 sregs->u.s.ppc64.slb[i].slbe = vcpu3s->slb[i].orige | i;
1194 sregs->u.s.ppc64.slb[i].slbv = vcpu3s->slb[i].origv;
1195 }
1196 } else {
1197 for (i = 0; i < 16; i++)
1198 sregs->u.s.ppc32.sr[i] = vcpu->arch.shared->sr[i];
1199
1200 for (i = 0; i < 8; i++) {
1201 sregs->u.s.ppc32.ibat[i] = vcpu3s->ibat[i].raw;
1202 sregs->u.s.ppc32.dbat[i] = vcpu3s->dbat[i].raw;
1203 }
1204 }
1205
1206 return 0;
1207}
1208
1209int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1210 struct kvm_sregs *sregs)
1211{
1212 struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu);
1213 int i;
1214
1215 kvmppc_set_pvr(vcpu, sregs->pvr);
1216
1217 vcpu3s->sdr1 = sregs->u.s.sdr1;
1218 if (vcpu->arch.hflags & BOOK3S_HFLAG_SLB) {
1219 for (i = 0; i < 64; i++) {
1220 vcpu->arch.mmu.slbmte(vcpu, sregs->u.s.ppc64.slb[i].slbv,
1221 sregs->u.s.ppc64.slb[i].slbe);
1222 }
1223 } else {
1224 for (i = 0; i < 16; i++) {
1225 vcpu->arch.mmu.mtsrin(vcpu, i, sregs->u.s.ppc32.sr[i]);
1226 }
1227 for (i = 0; i < 8; i++) {
1228 kvmppc_set_bat(vcpu, &(vcpu3s->ibat[i]), false,
1229 (u32)sregs->u.s.ppc32.ibat[i]);
1230 kvmppc_set_bat(vcpu, &(vcpu3s->ibat[i]), true,
1231 (u32)(sregs->u.s.ppc32.ibat[i] >> 32));
1232 kvmppc_set_bat(vcpu, &(vcpu3s->dbat[i]), false,
1233 (u32)sregs->u.s.ppc32.dbat[i]);
1234 kvmppc_set_bat(vcpu, &(vcpu3s->dbat[i]), true,
1235 (u32)(sregs->u.s.ppc32.dbat[i] >> 32));
1236 }
1237 }
1238
1239 /* Flush the MMU after messing with the segments */
1240 kvmppc_mmu_pte_flush(vcpu, 0, 0);
1241
1242 return 0;
1243}
1244
1245int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 463int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1246{ 464{
1247 return -ENOTSUPP; 465 return -ENOTSUPP;
@@ -1296,202 +514,3 @@ out:
1296 mutex_unlock(&kvm->slots_lock); 514 mutex_unlock(&kvm->slots_lock);
1297 return r; 515 return r;
1298} 516}
1299
1300int kvmppc_core_check_processor_compat(void)
1301{
1302 return 0;
1303}
1304
1305struct kvm_vcpu *kvmppc_core_vcpu_create(struct kvm *kvm, unsigned int id)
1306{
1307 struct kvmppc_vcpu_book3s *vcpu_book3s;
1308 struct kvm_vcpu *vcpu;
1309 int err = -ENOMEM;
1310 unsigned long p;
1311
1312 vcpu_book3s = vzalloc(sizeof(struct kvmppc_vcpu_book3s));
1313 if (!vcpu_book3s)
1314 goto out;
1315
1316 vcpu_book3s->shadow_vcpu = (struct kvmppc_book3s_shadow_vcpu *)
1317 kzalloc(sizeof(*vcpu_book3s->shadow_vcpu), GFP_KERNEL);
1318 if (!vcpu_book3s->shadow_vcpu)
1319 goto free_vcpu;
1320
1321 vcpu = &vcpu_book3s->vcpu;
1322 err = kvm_vcpu_init(vcpu, kvm, id);
1323 if (err)
1324 goto free_shadow_vcpu;
1325
1326 p = __get_free_page(GFP_KERNEL|__GFP_ZERO);
1327 /* the real shared page fills the last 4k of our page */
1328 vcpu->arch.shared = (void*)(p + PAGE_SIZE - 4096);
1329 if (!p)
1330 goto uninit_vcpu;
1331
1332 vcpu->arch.host_retip = kvm_return_point;
1333 vcpu->arch.host_msr = mfmsr();
1334#ifdef CONFIG_PPC_BOOK3S_64
1335 /* default to book3s_64 (970fx) */
1336 vcpu->arch.pvr = 0x3C0301;
1337#else
1338 /* default to book3s_32 (750) */
1339 vcpu->arch.pvr = 0x84202;
1340#endif
1341 kvmppc_set_pvr(vcpu, vcpu->arch.pvr);
1342 vcpu_book3s->slb_nr = 64;
1343
1344 /* remember where some real-mode handlers are */
1345 vcpu->arch.trampoline_lowmem = kvmppc_trampoline_lowmem;
1346 vcpu->arch.trampoline_enter = kvmppc_trampoline_enter;
1347 vcpu->arch.highmem_handler = (ulong)kvmppc_handler_highmem;
1348#ifdef CONFIG_PPC_BOOK3S_64
1349 vcpu->arch.rmcall = *(ulong*)kvmppc_rmcall;
1350#else
1351 vcpu->arch.rmcall = (ulong)kvmppc_rmcall;
1352#endif
1353
1354 vcpu->arch.shadow_msr = MSR_USER64;
1355
1356 err = kvmppc_mmu_init(vcpu);
1357 if (err < 0)
1358 goto uninit_vcpu;
1359
1360 return vcpu;
1361
1362uninit_vcpu:
1363 kvm_vcpu_uninit(vcpu);
1364free_shadow_vcpu:
1365 kfree(vcpu_book3s->shadow_vcpu);
1366free_vcpu:
1367 vfree(vcpu_book3s);
1368out:
1369 return ERR_PTR(err);
1370}
1371
1372void kvmppc_core_vcpu_free(struct kvm_vcpu *vcpu)
1373{
1374 struct kvmppc_vcpu_book3s *vcpu_book3s = to_book3s(vcpu);
1375
1376 free_page((unsigned long)vcpu->arch.shared & PAGE_MASK);
1377 kvm_vcpu_uninit(vcpu);
1378 kfree(vcpu_book3s->shadow_vcpu);
1379 vfree(vcpu_book3s);
1380}
1381
1382extern int __kvmppc_vcpu_entry(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu);
1383int __kvmppc_vcpu_run(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
1384{
1385 int ret;
1386 double fpr[32][TS_FPRWIDTH];
1387 unsigned int fpscr;
1388 int fpexc_mode;
1389#ifdef CONFIG_ALTIVEC
1390 vector128 vr[32];
1391 vector128 vscr;
1392 unsigned long uninitialized_var(vrsave);
1393 int used_vr;
1394#endif
1395#ifdef CONFIG_VSX
1396 int used_vsr;
1397#endif
1398 ulong ext_msr;
1399
1400 /* No need to go into the guest when all we do is going out */
1401 if (signal_pending(current)) {
1402 kvm_run->exit_reason = KVM_EXIT_INTR;
1403 return -EINTR;
1404 }
1405
1406 /* Save FPU state in stack */
1407 if (current->thread.regs->msr & MSR_FP)
1408 giveup_fpu(current);
1409 memcpy(fpr, current->thread.fpr, sizeof(current->thread.fpr));
1410 fpscr = current->thread.fpscr.val;
1411 fpexc_mode = current->thread.fpexc_mode;
1412
1413#ifdef CONFIG_ALTIVEC
1414 /* Save Altivec state in stack */
1415 used_vr = current->thread.used_vr;
1416 if (used_vr) {
1417 if (current->thread.regs->msr & MSR_VEC)
1418 giveup_altivec(current);
1419 memcpy(vr, current->thread.vr, sizeof(current->thread.vr));
1420 vscr = current->thread.vscr;
1421 vrsave = current->thread.vrsave;
1422 }
1423#endif
1424
1425#ifdef CONFIG_VSX
1426 /* Save VSX state in stack */
1427 used_vsr = current->thread.used_vsr;
1428 if (used_vsr && (current->thread.regs->msr & MSR_VSX))
1429 __giveup_vsx(current);
1430#endif
1431
1432 /* Remember the MSR with disabled extensions */
1433 ext_msr = current->thread.regs->msr;
1434
1435 /* XXX we get called with irq disabled - change that! */
1436 local_irq_enable();
1437
1438 /* Preload FPU if it's enabled */
1439 if (vcpu->arch.shared->msr & MSR_FP)
1440 kvmppc_handle_ext(vcpu, BOOK3S_INTERRUPT_FP_UNAVAIL, MSR_FP);
1441
1442 ret = __kvmppc_vcpu_entry(kvm_run, vcpu);
1443
1444 local_irq_disable();
1445
1446 current->thread.regs->msr = ext_msr;
1447
1448 /* Make sure we save the guest FPU/Altivec/VSX state */
1449 kvmppc_giveup_ext(vcpu, MSR_FP);
1450 kvmppc_giveup_ext(vcpu, MSR_VEC);
1451 kvmppc_giveup_ext(vcpu, MSR_VSX);
1452
1453 /* Restore FPU state from stack */
1454 memcpy(current->thread.fpr, fpr, sizeof(current->thread.fpr));
1455 current->thread.fpscr.val = fpscr;
1456 current->thread.fpexc_mode = fpexc_mode;
1457
1458#ifdef CONFIG_ALTIVEC
1459 /* Restore Altivec state from stack */
1460 if (used_vr && current->thread.used_vr) {
1461 memcpy(current->thread.vr, vr, sizeof(current->thread.vr));
1462 current->thread.vscr = vscr;
1463 current->thread.vrsave = vrsave;
1464 }
1465 current->thread.used_vr = used_vr;
1466#endif
1467
1468#ifdef CONFIG_VSX
1469 current->thread.used_vsr = used_vsr;
1470#endif
1471
1472 return ret;
1473}
1474
1475static int kvmppc_book3s_init(void)
1476{
1477 int r;
1478
1479 r = kvm_init(NULL, sizeof(struct kvmppc_vcpu_book3s), 0,
1480 THIS_MODULE);
1481
1482 if (r)
1483 return r;
1484
1485 r = kvmppc_mmu_hpte_sysinit();
1486
1487 return r;
1488}
1489
1490static void kvmppc_book3s_exit(void)
1491{
1492 kvmppc_mmu_hpte_sysexit();
1493 kvm_exit();
1494}
1495
1496module_init(kvmppc_book3s_init);
1497module_exit(kvmppc_book3s_exit);
diff --git a/arch/powerpc/kvm/book3s_64_mmu.c b/arch/powerpc/kvm/book3s_64_mmu.c
index d7889ef3211e..c6d3e194b6b4 100644
--- a/arch/powerpc/kvm/book3s_64_mmu.c
+++ b/arch/powerpc/kvm/book3s_64_mmu.c
@@ -41,36 +41,36 @@ static void kvmppc_mmu_book3s_64_reset_msr(struct kvm_vcpu *vcpu)
41} 41}
42 42
43static struct kvmppc_slb *kvmppc_mmu_book3s_64_find_slbe( 43static struct kvmppc_slb *kvmppc_mmu_book3s_64_find_slbe(
44 struct kvmppc_vcpu_book3s *vcpu_book3s, 44 struct kvm_vcpu *vcpu,
45 gva_t eaddr) 45 gva_t eaddr)
46{ 46{
47 int i; 47 int i;
48 u64 esid = GET_ESID(eaddr); 48 u64 esid = GET_ESID(eaddr);
49 u64 esid_1t = GET_ESID_1T(eaddr); 49 u64 esid_1t = GET_ESID_1T(eaddr);
50 50
51 for (i = 0; i < vcpu_book3s->slb_nr; i++) { 51 for (i = 0; i < vcpu->arch.slb_nr; i++) {
52 u64 cmp_esid = esid; 52 u64 cmp_esid = esid;
53 53
54 if (!vcpu_book3s->slb[i].valid) 54 if (!vcpu->arch.slb[i].valid)
55 continue; 55 continue;
56 56
57 if (vcpu_book3s->slb[i].tb) 57 if (vcpu->arch.slb[i].tb)
58 cmp_esid = esid_1t; 58 cmp_esid = esid_1t;
59 59
60 if (vcpu_book3s->slb[i].esid == cmp_esid) 60 if (vcpu->arch.slb[i].esid == cmp_esid)
61 return &vcpu_book3s->slb[i]; 61 return &vcpu->arch.slb[i];
62 } 62 }
63 63
64 dprintk("KVM: No SLB entry found for 0x%lx [%llx | %llx]\n", 64 dprintk("KVM: No SLB entry found for 0x%lx [%llx | %llx]\n",
65 eaddr, esid, esid_1t); 65 eaddr, esid, esid_1t);
66 for (i = 0; i < vcpu_book3s->slb_nr; i++) { 66 for (i = 0; i < vcpu->arch.slb_nr; i++) {
67 if (vcpu_book3s->slb[i].vsid) 67 if (vcpu->arch.slb[i].vsid)
68 dprintk(" %d: %c%c%c %llx %llx\n", i, 68 dprintk(" %d: %c%c%c %llx %llx\n", i,
69 vcpu_book3s->slb[i].valid ? 'v' : ' ', 69 vcpu->arch.slb[i].valid ? 'v' : ' ',
70 vcpu_book3s->slb[i].large ? 'l' : ' ', 70 vcpu->arch.slb[i].large ? 'l' : ' ',
71 vcpu_book3s->slb[i].tb ? 't' : ' ', 71 vcpu->arch.slb[i].tb ? 't' : ' ',
72 vcpu_book3s->slb[i].esid, 72 vcpu->arch.slb[i].esid,
73 vcpu_book3s->slb[i].vsid); 73 vcpu->arch.slb[i].vsid);
74 } 74 }
75 75
76 return NULL; 76 return NULL;
@@ -81,7 +81,7 @@ static u64 kvmppc_mmu_book3s_64_ea_to_vp(struct kvm_vcpu *vcpu, gva_t eaddr,
81{ 81{
82 struct kvmppc_slb *slb; 82 struct kvmppc_slb *slb;
83 83
84 slb = kvmppc_mmu_book3s_64_find_slbe(to_book3s(vcpu), eaddr); 84 slb = kvmppc_mmu_book3s_64_find_slbe(vcpu, eaddr);
85 if (!slb) 85 if (!slb)
86 return 0; 86 return 0;
87 87
@@ -180,7 +180,7 @@ static int kvmppc_mmu_book3s_64_xlate(struct kvm_vcpu *vcpu, gva_t eaddr,
180 return 0; 180 return 0;
181 } 181 }
182 182
183 slbe = kvmppc_mmu_book3s_64_find_slbe(vcpu_book3s, eaddr); 183 slbe = kvmppc_mmu_book3s_64_find_slbe(vcpu, eaddr);
184 if (!slbe) 184 if (!slbe)
185 goto no_seg_found; 185 goto no_seg_found;
186 186
@@ -320,10 +320,10 @@ static void kvmppc_mmu_book3s_64_slbmte(struct kvm_vcpu *vcpu, u64 rs, u64 rb)
320 esid_1t = GET_ESID_1T(rb); 320 esid_1t = GET_ESID_1T(rb);
321 slb_nr = rb & 0xfff; 321 slb_nr = rb & 0xfff;
322 322
323 if (slb_nr > vcpu_book3s->slb_nr) 323 if (slb_nr > vcpu->arch.slb_nr)
324 return; 324 return;
325 325
326 slbe = &vcpu_book3s->slb[slb_nr]; 326 slbe = &vcpu->arch.slb[slb_nr];
327 327
328 slbe->large = (rs & SLB_VSID_L) ? 1 : 0; 328 slbe->large = (rs & SLB_VSID_L) ? 1 : 0;
329 slbe->tb = (rs & SLB_VSID_B_1T) ? 1 : 0; 329 slbe->tb = (rs & SLB_VSID_B_1T) ? 1 : 0;
@@ -344,38 +344,35 @@ static void kvmppc_mmu_book3s_64_slbmte(struct kvm_vcpu *vcpu, u64 rs, u64 rb)
344 344
345static u64 kvmppc_mmu_book3s_64_slbmfee(struct kvm_vcpu *vcpu, u64 slb_nr) 345static u64 kvmppc_mmu_book3s_64_slbmfee(struct kvm_vcpu *vcpu, u64 slb_nr)
346{ 346{
347 struct kvmppc_vcpu_book3s *vcpu_book3s = to_book3s(vcpu);
348 struct kvmppc_slb *slbe; 347 struct kvmppc_slb *slbe;
349 348
350 if (slb_nr > vcpu_book3s->slb_nr) 349 if (slb_nr > vcpu->arch.slb_nr)
351 return 0; 350 return 0;
352 351
353 slbe = &vcpu_book3s->slb[slb_nr]; 352 slbe = &vcpu->arch.slb[slb_nr];
354 353
355 return slbe->orige; 354 return slbe->orige;
356} 355}
357 356
358static u64 kvmppc_mmu_book3s_64_slbmfev(struct kvm_vcpu *vcpu, u64 slb_nr) 357static u64 kvmppc_mmu_book3s_64_slbmfev(struct kvm_vcpu *vcpu, u64 slb_nr)
359{ 358{
360 struct kvmppc_vcpu_book3s *vcpu_book3s = to_book3s(vcpu);
361 struct kvmppc_slb *slbe; 359 struct kvmppc_slb *slbe;
362 360
363 if (slb_nr > vcpu_book3s->slb_nr) 361 if (slb_nr > vcpu->arch.slb_nr)
364 return 0; 362 return 0;
365 363
366 slbe = &vcpu_book3s->slb[slb_nr]; 364 slbe = &vcpu->arch.slb[slb_nr];
367 365
368 return slbe->origv; 366 return slbe->origv;
369} 367}
370 368
371static void kvmppc_mmu_book3s_64_slbie(struct kvm_vcpu *vcpu, u64 ea) 369static void kvmppc_mmu_book3s_64_slbie(struct kvm_vcpu *vcpu, u64 ea)
372{ 370{
373 struct kvmppc_vcpu_book3s *vcpu_book3s = to_book3s(vcpu);
374 struct kvmppc_slb *slbe; 371 struct kvmppc_slb *slbe;
375 372
376 dprintk("KVM MMU: slbie(0x%llx)\n", ea); 373 dprintk("KVM MMU: slbie(0x%llx)\n", ea);
377 374
378 slbe = kvmppc_mmu_book3s_64_find_slbe(vcpu_book3s, ea); 375 slbe = kvmppc_mmu_book3s_64_find_slbe(vcpu, ea);
379 376
380 if (!slbe) 377 if (!slbe)
381 return; 378 return;
@@ -389,13 +386,12 @@ static void kvmppc_mmu_book3s_64_slbie(struct kvm_vcpu *vcpu, u64 ea)
389 386
390static void kvmppc_mmu_book3s_64_slbia(struct kvm_vcpu *vcpu) 387static void kvmppc_mmu_book3s_64_slbia(struct kvm_vcpu *vcpu)
391{ 388{
392 struct kvmppc_vcpu_book3s *vcpu_book3s = to_book3s(vcpu);
393 int i; 389 int i;
394 390
395 dprintk("KVM MMU: slbia()\n"); 391 dprintk("KVM MMU: slbia()\n");
396 392
397 for (i = 1; i < vcpu_book3s->slb_nr; i++) 393 for (i = 1; i < vcpu->arch.slb_nr; i++)
398 vcpu_book3s->slb[i].valid = false; 394 vcpu->arch.slb[i].valid = false;
399 395
400 if (vcpu->arch.shared->msr & MSR_IR) { 396 if (vcpu->arch.shared->msr & MSR_IR) {
401 kvmppc_mmu_flush_segments(vcpu); 397 kvmppc_mmu_flush_segments(vcpu);
@@ -464,7 +460,7 @@ static int kvmppc_mmu_book3s_64_esid_to_vsid(struct kvm_vcpu *vcpu, ulong esid,
464 ulong mp_ea = vcpu->arch.magic_page_ea; 460 ulong mp_ea = vcpu->arch.magic_page_ea;
465 461
466 if (vcpu->arch.shared->msr & (MSR_DR|MSR_IR)) { 462 if (vcpu->arch.shared->msr & (MSR_DR|MSR_IR)) {
467 slb = kvmppc_mmu_book3s_64_find_slbe(to_book3s(vcpu), ea); 463 slb = kvmppc_mmu_book3s_64_find_slbe(vcpu, ea);
468 if (slb) 464 if (slb)
469 gvsid = slb->vsid; 465 gvsid = slb->vsid;
470 } 466 }
diff --git a/arch/powerpc/kvm/book3s_64_mmu_hv.c b/arch/powerpc/kvm/book3s_64_mmu_hv.c
new file mode 100644
index 000000000000..bc3a2ea94217
--- /dev/null
+++ b/arch/powerpc/kvm/book3s_64_mmu_hv.c
@@ -0,0 +1,180 @@
1/*
2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License, version 2, as
4 * published by the Free Software Foundation.
5 *
6 * This program is distributed in the hope that it will be useful,
7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9 * GNU General Public License for more details.
10 *
11 * You should have received a copy of the GNU General Public License
12 * along with this program; if not, write to the Free Software
13 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
14 *
15 * Copyright 2010 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
16 */
17
18#include <linux/types.h>
19#include <linux/string.h>
20#include <linux/kvm.h>
21#include <linux/kvm_host.h>
22#include <linux/highmem.h>
23#include <linux/gfp.h>
24#include <linux/slab.h>
25#include <linux/hugetlb.h>
26
27#include <asm/tlbflush.h>
28#include <asm/kvm_ppc.h>
29#include <asm/kvm_book3s.h>
30#include <asm/mmu-hash64.h>
31#include <asm/hvcall.h>
32#include <asm/synch.h>
33#include <asm/ppc-opcode.h>
34#include <asm/cputable.h>
35
36/* For now use fixed-size 16MB page table */
37#define HPT_ORDER 24
38#define HPT_NPTEG (1ul << (HPT_ORDER - 7)) /* 128B per pteg */
39#define HPT_HASH_MASK (HPT_NPTEG - 1)
40
41/* Pages in the VRMA are 16MB pages */
42#define VRMA_PAGE_ORDER 24
43#define VRMA_VSID 0x1ffffffUL /* 1TB VSID reserved for VRMA */
44
45/* POWER7 has 10-bit LPIDs, PPC970 has 6-bit LPIDs */
46#define MAX_LPID_970 63
47#define NR_LPIDS (LPID_RSVD + 1)
48unsigned long lpid_inuse[BITS_TO_LONGS(NR_LPIDS)];
49
50long kvmppc_alloc_hpt(struct kvm *kvm)
51{
52 unsigned long hpt;
53 unsigned long lpid;
54
55 hpt = __get_free_pages(GFP_KERNEL|__GFP_ZERO|__GFP_REPEAT|__GFP_NOWARN,
56 HPT_ORDER - PAGE_SHIFT);
57 if (!hpt) {
58 pr_err("kvm_alloc_hpt: Couldn't alloc HPT\n");
59 return -ENOMEM;
60 }
61 kvm->arch.hpt_virt = hpt;
62
63 do {
64 lpid = find_first_zero_bit(lpid_inuse, NR_LPIDS);
65 if (lpid >= NR_LPIDS) {
66 pr_err("kvm_alloc_hpt: No LPIDs free\n");
67 free_pages(hpt, HPT_ORDER - PAGE_SHIFT);
68 return -ENOMEM;
69 }
70 } while (test_and_set_bit(lpid, lpid_inuse));
71
72 kvm->arch.sdr1 = __pa(hpt) | (HPT_ORDER - 18);
73 kvm->arch.lpid = lpid;
74
75 pr_info("KVM guest htab at %lx, LPID %lx\n", hpt, lpid);
76 return 0;
77}
78
79void kvmppc_free_hpt(struct kvm *kvm)
80{
81 clear_bit(kvm->arch.lpid, lpid_inuse);
82 free_pages(kvm->arch.hpt_virt, HPT_ORDER - PAGE_SHIFT);
83}
84
85void kvmppc_map_vrma(struct kvm *kvm, struct kvm_userspace_memory_region *mem)
86{
87 unsigned long i;
88 unsigned long npages = kvm->arch.ram_npages;
89 unsigned long pfn;
90 unsigned long *hpte;
91 unsigned long hash;
92 struct kvmppc_pginfo *pginfo = kvm->arch.ram_pginfo;
93
94 if (!pginfo)
95 return;
96
97 /* VRMA can't be > 1TB */
98 if (npages > 1ul << (40 - kvm->arch.ram_porder))
99 npages = 1ul << (40 - kvm->arch.ram_porder);
100 /* Can't use more than 1 HPTE per HPTEG */
101 if (npages > HPT_NPTEG)
102 npages = HPT_NPTEG;
103
104 for (i = 0; i < npages; ++i) {
105 pfn = pginfo[i].pfn;
106 if (!pfn)
107 break;
108 /* can't use hpt_hash since va > 64 bits */
109 hash = (i ^ (VRMA_VSID ^ (VRMA_VSID << 25))) & HPT_HASH_MASK;
110 /*
111 * We assume that the hash table is empty and no
112 * vcpus are using it at this stage. Since we create
113 * at most one HPTE per HPTEG, we just assume entry 7
114 * is available and use it.
115 */
116 hpte = (unsigned long *) (kvm->arch.hpt_virt + (hash << 7));
117 hpte += 7 * 2;
118 /* HPTE low word - RPN, protection, etc. */
119 hpte[1] = (pfn << PAGE_SHIFT) | HPTE_R_R | HPTE_R_C |
120 HPTE_R_M | PP_RWXX;
121 wmb();
122 hpte[0] = HPTE_V_1TB_SEG | (VRMA_VSID << (40 - 16)) |
123 (i << (VRMA_PAGE_ORDER - 16)) | HPTE_V_BOLTED |
124 HPTE_V_LARGE | HPTE_V_VALID;
125 }
126}
127
128int kvmppc_mmu_hv_init(void)
129{
130 unsigned long host_lpid, rsvd_lpid;
131
132 if (!cpu_has_feature(CPU_FTR_HVMODE))
133 return -EINVAL;
134
135 memset(lpid_inuse, 0, sizeof(lpid_inuse));
136
137 if (cpu_has_feature(CPU_FTR_ARCH_206)) {
138 host_lpid = mfspr(SPRN_LPID); /* POWER7 */
139 rsvd_lpid = LPID_RSVD;
140 } else {
141 host_lpid = 0; /* PPC970 */
142 rsvd_lpid = MAX_LPID_970;
143 }
144
145 set_bit(host_lpid, lpid_inuse);
146 /* rsvd_lpid is reserved for use in partition switching */
147 set_bit(rsvd_lpid, lpid_inuse);
148
149 return 0;
150}
151
152void kvmppc_mmu_destroy(struct kvm_vcpu *vcpu)
153{
154}
155
156static void kvmppc_mmu_book3s_64_hv_reset_msr(struct kvm_vcpu *vcpu)
157{
158 kvmppc_set_msr(vcpu, MSR_SF | MSR_ME);
159}
160
161static int kvmppc_mmu_book3s_64_hv_xlate(struct kvm_vcpu *vcpu, gva_t eaddr,
162 struct kvmppc_pte *gpte, bool data)
163{
164 return -ENOENT;
165}
166
167void kvmppc_mmu_book3s_hv_init(struct kvm_vcpu *vcpu)
168{
169 struct kvmppc_mmu *mmu = &vcpu->arch.mmu;
170
171 if (cpu_has_feature(CPU_FTR_ARCH_206))
172 vcpu->arch.slb_nr = 32; /* POWER7 */
173 else
174 vcpu->arch.slb_nr = 64;
175
176 mmu->xlate = kvmppc_mmu_book3s_64_hv_xlate;
177 mmu->reset_msr = kvmppc_mmu_book3s_64_hv_reset_msr;
178
179 vcpu->arch.hflags |= BOOK3S_HFLAG_SLB;
180}
diff --git a/arch/powerpc/kvm/book3s_64_vio_hv.c b/arch/powerpc/kvm/book3s_64_vio_hv.c
new file mode 100644
index 000000000000..ea0f8c537c28
--- /dev/null
+++ b/arch/powerpc/kvm/book3s_64_vio_hv.c
@@ -0,0 +1,73 @@
1/*
2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License, version 2, as
4 * published by the Free Software Foundation.
5 *
6 * This program is distributed in the hope that it will be useful,
7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9 * GNU General Public License for more details.
10 *
11 * You should have received a copy of the GNU General Public License
12 * along with this program; if not, write to the Free Software
13 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
14 *
15 * Copyright 2010 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
16 * Copyright 2011 David Gibson, IBM Corporation <dwg@au1.ibm.com>
17 */
18
19#include <linux/types.h>
20#include <linux/string.h>
21#include <linux/kvm.h>
22#include <linux/kvm_host.h>
23#include <linux/highmem.h>
24#include <linux/gfp.h>
25#include <linux/slab.h>
26#include <linux/hugetlb.h>
27#include <linux/list.h>
28
29#include <asm/tlbflush.h>
30#include <asm/kvm_ppc.h>
31#include <asm/kvm_book3s.h>
32#include <asm/mmu-hash64.h>
33#include <asm/hvcall.h>
34#include <asm/synch.h>
35#include <asm/ppc-opcode.h>
36#include <asm/kvm_host.h>
37#include <asm/udbg.h>
38
39#define TCES_PER_PAGE (PAGE_SIZE / sizeof(u64))
40
41long kvmppc_h_put_tce(struct kvm_vcpu *vcpu, unsigned long liobn,
42 unsigned long ioba, unsigned long tce)
43{
44 struct kvm *kvm = vcpu->kvm;
45 struct kvmppc_spapr_tce_table *stt;
46
47 /* udbg_printf("H_PUT_TCE(): liobn=0x%lx ioba=0x%lx, tce=0x%lx\n", */
48 /* liobn, ioba, tce); */
49
50 list_for_each_entry(stt, &kvm->arch.spapr_tce_tables, list) {
51 if (stt->liobn == liobn) {
52 unsigned long idx = ioba >> SPAPR_TCE_SHIFT;
53 struct page *page;
54 u64 *tbl;
55
56 /* udbg_printf("H_PUT_TCE: liobn 0x%lx => stt=%p window_size=0x%x\n", */
57 /* liobn, stt, stt->window_size); */
58 if (ioba >= stt->window_size)
59 return H_PARAMETER;
60
61 page = stt->pages[idx / TCES_PER_PAGE];
62 tbl = (u64 *)page_address(page);
63
64 /* FIXME: Need to validate the TCE itself */
65 /* udbg_printf("tce @ %p\n", &tbl[idx % TCES_PER_PAGE]); */
66 tbl[idx % TCES_PER_PAGE] = tce;
67 return H_SUCCESS;
68 }
69 }
70
71 /* Didn't find the liobn, punt it to userspace */
72 return H_TOO_HARD;
73}
diff --git a/arch/powerpc/kvm/book3s_exports.c b/arch/powerpc/kvm/book3s_exports.c
index 1dd5a1ddfd0d..88c8f26add02 100644
--- a/arch/powerpc/kvm/book3s_exports.c
+++ b/arch/powerpc/kvm/book3s_exports.c
@@ -20,8 +20,11 @@
20#include <linux/module.h> 20#include <linux/module.h>
21#include <asm/kvm_book3s.h> 21#include <asm/kvm_book3s.h>
22 22
23EXPORT_SYMBOL_GPL(kvmppc_trampoline_enter); 23#ifdef CONFIG_KVM_BOOK3S_64_HV
24EXPORT_SYMBOL_GPL(kvmppc_trampoline_lowmem); 24EXPORT_SYMBOL_GPL(kvmppc_hv_entry_trampoline);
25#else
26EXPORT_SYMBOL_GPL(kvmppc_handler_trampoline_enter);
27EXPORT_SYMBOL_GPL(kvmppc_handler_lowmem_trampoline);
25EXPORT_SYMBOL_GPL(kvmppc_rmcall); 28EXPORT_SYMBOL_GPL(kvmppc_rmcall);
26EXPORT_SYMBOL_GPL(kvmppc_load_up_fpu); 29EXPORT_SYMBOL_GPL(kvmppc_load_up_fpu);
27#ifdef CONFIG_ALTIVEC 30#ifdef CONFIG_ALTIVEC
@@ -30,3 +33,5 @@ EXPORT_SYMBOL_GPL(kvmppc_load_up_altivec);
30#ifdef CONFIG_VSX 33#ifdef CONFIG_VSX
31EXPORT_SYMBOL_GPL(kvmppc_load_up_vsx); 34EXPORT_SYMBOL_GPL(kvmppc_load_up_vsx);
32#endif 35#endif
36#endif
37
diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c
new file mode 100644
index 000000000000..cc0d7f1b19ab
--- /dev/null
+++ b/arch/powerpc/kvm/book3s_hv.c
@@ -0,0 +1,1269 @@
1/*
2 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
3 * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved.
4 *
5 * Authors:
6 * Paul Mackerras <paulus@au1.ibm.com>
7 * Alexander Graf <agraf@suse.de>
8 * Kevin Wolf <mail@kevin-wolf.de>
9 *
10 * Description: KVM functions specific to running on Book 3S
11 * processors in hypervisor mode (specifically POWER7 and later).
12 *
13 * This file is derived from arch/powerpc/kvm/book3s.c,
14 * by Alexander Graf <agraf@suse.de>.
15 *
16 * This program is free software; you can redistribute it and/or modify
17 * it under the terms of the GNU General Public License, version 2, as
18 * published by the Free Software Foundation.
19 */
20
21#include <linux/kvm_host.h>
22#include <linux/err.h>
23#include <linux/slab.h>
24#include <linux/preempt.h>
25#include <linux/sched.h>
26#include <linux/delay.h>
27#include <linux/fs.h>
28#include <linux/anon_inodes.h>
29#include <linux/cpumask.h>
30#include <linux/spinlock.h>
31#include <linux/page-flags.h>
32
33#include <asm/reg.h>
34#include <asm/cputable.h>
35#include <asm/cacheflush.h>
36#include <asm/tlbflush.h>
37#include <asm/uaccess.h>
38#include <asm/io.h>
39#include <asm/kvm_ppc.h>
40#include <asm/kvm_book3s.h>
41#include <asm/mmu_context.h>
42#include <asm/lppaca.h>
43#include <asm/processor.h>
44#include <asm/cputhreads.h>
45#include <asm/page.h>
46#include <linux/gfp.h>
47#include <linux/sched.h>
48#include <linux/vmalloc.h>
49#include <linux/highmem.h>
50
51/*
52 * For now, limit memory to 64GB and require it to be large pages.
53 * This value is chosen because it makes the ram_pginfo array be
54 * 64kB in size, which is about as large as we want to be trying
55 * to allocate with kmalloc.
56 */
57#define MAX_MEM_ORDER 36
58
59#define LARGE_PAGE_ORDER 24 /* 16MB pages */
60
61/* #define EXIT_DEBUG */
62/* #define EXIT_DEBUG_SIMPLE */
63/* #define EXIT_DEBUG_INT */
64
65void kvmppc_core_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
66{
67 local_paca->kvm_hstate.kvm_vcpu = vcpu;
68 local_paca->kvm_hstate.kvm_vcore = vcpu->arch.vcore;
69}
70
71void kvmppc_core_vcpu_put(struct kvm_vcpu *vcpu)
72{
73}
74
75static void kvmppc_vcpu_blocked(struct kvm_vcpu *vcpu);
76static void kvmppc_vcpu_unblocked(struct kvm_vcpu *vcpu);
77
78void kvmppc_vcpu_block(struct kvm_vcpu *vcpu)
79{
80 u64 now;
81 unsigned long dec_nsec;
82
83 now = get_tb();
84 if (now >= vcpu->arch.dec_expires && !kvmppc_core_pending_dec(vcpu))
85 kvmppc_core_queue_dec(vcpu);
86 if (vcpu->arch.pending_exceptions)
87 return;
88 if (vcpu->arch.dec_expires != ~(u64)0) {
89 dec_nsec = (vcpu->arch.dec_expires - now) * NSEC_PER_SEC /
90 tb_ticks_per_sec;
91 hrtimer_start(&vcpu->arch.dec_timer, ktime_set(0, dec_nsec),
92 HRTIMER_MODE_REL);
93 }
94
95 kvmppc_vcpu_blocked(vcpu);
96
97 kvm_vcpu_block(vcpu);
98 vcpu->stat.halt_wakeup++;
99
100 if (vcpu->arch.dec_expires != ~(u64)0)
101 hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
102
103 kvmppc_vcpu_unblocked(vcpu);
104}
105
106void kvmppc_set_msr(struct kvm_vcpu *vcpu, u64 msr)
107{
108 vcpu->arch.shregs.msr = msr;
109}
110
111void kvmppc_set_pvr(struct kvm_vcpu *vcpu, u32 pvr)
112{
113 vcpu->arch.pvr = pvr;
114}
115
116void kvmppc_dump_regs(struct kvm_vcpu *vcpu)
117{
118 int r;
119
120 pr_err("vcpu %p (%d):\n", vcpu, vcpu->vcpu_id);
121 pr_err("pc = %.16lx msr = %.16llx trap = %x\n",
122 vcpu->arch.pc, vcpu->arch.shregs.msr, vcpu->arch.trap);
123 for (r = 0; r < 16; ++r)
124 pr_err("r%2d = %.16lx r%d = %.16lx\n",
125 r, kvmppc_get_gpr(vcpu, r),
126 r+16, kvmppc_get_gpr(vcpu, r+16));
127 pr_err("ctr = %.16lx lr = %.16lx\n",
128 vcpu->arch.ctr, vcpu->arch.lr);
129 pr_err("srr0 = %.16llx srr1 = %.16llx\n",
130 vcpu->arch.shregs.srr0, vcpu->arch.shregs.srr1);
131 pr_err("sprg0 = %.16llx sprg1 = %.16llx\n",
132 vcpu->arch.shregs.sprg0, vcpu->arch.shregs.sprg1);
133 pr_err("sprg2 = %.16llx sprg3 = %.16llx\n",
134 vcpu->arch.shregs.sprg2, vcpu->arch.shregs.sprg3);
135 pr_err("cr = %.8x xer = %.16lx dsisr = %.8x\n",
136 vcpu->arch.cr, vcpu->arch.xer, vcpu->arch.shregs.dsisr);
137 pr_err("dar = %.16llx\n", vcpu->arch.shregs.dar);
138 pr_err("fault dar = %.16lx dsisr = %.8x\n",
139 vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
140 pr_err("SLB (%d entries):\n", vcpu->arch.slb_max);
141 for (r = 0; r < vcpu->arch.slb_max; ++r)
142 pr_err(" ESID = %.16llx VSID = %.16llx\n",
143 vcpu->arch.slb[r].orige, vcpu->arch.slb[r].origv);
144 pr_err("lpcr = %.16lx sdr1 = %.16lx last_inst = %.8x\n",
145 vcpu->kvm->arch.lpcr, vcpu->kvm->arch.sdr1,
146 vcpu->arch.last_inst);
147}
148
149struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id)
150{
151 int r;
152 struct kvm_vcpu *v, *ret = NULL;
153
154 mutex_lock(&kvm->lock);
155 kvm_for_each_vcpu(r, v, kvm) {
156 if (v->vcpu_id == id) {
157 ret = v;
158 break;
159 }
160 }
161 mutex_unlock(&kvm->lock);
162 return ret;
163}
164
165static void init_vpa(struct kvm_vcpu *vcpu, struct lppaca *vpa)
166{
167 vpa->shared_proc = 1;
168 vpa->yield_count = 1;
169}
170
171static unsigned long do_h_register_vpa(struct kvm_vcpu *vcpu,
172 unsigned long flags,
173 unsigned long vcpuid, unsigned long vpa)
174{
175 struct kvm *kvm = vcpu->kvm;
176 unsigned long pg_index, ra, len;
177 unsigned long pg_offset;
178 void *va;
179 struct kvm_vcpu *tvcpu;
180
181 tvcpu = kvmppc_find_vcpu(kvm, vcpuid);
182 if (!tvcpu)
183 return H_PARAMETER;
184
185 flags >>= 63 - 18;
186 flags &= 7;
187 if (flags == 0 || flags == 4)
188 return H_PARAMETER;
189 if (flags < 4) {
190 if (vpa & 0x7f)
191 return H_PARAMETER;
192 /* registering new area; convert logical addr to real */
193 pg_index = vpa >> kvm->arch.ram_porder;
194 pg_offset = vpa & (kvm->arch.ram_psize - 1);
195 if (pg_index >= kvm->arch.ram_npages)
196 return H_PARAMETER;
197 if (kvm->arch.ram_pginfo[pg_index].pfn == 0)
198 return H_PARAMETER;
199 ra = kvm->arch.ram_pginfo[pg_index].pfn << PAGE_SHIFT;
200 ra |= pg_offset;
201 va = __va(ra);
202 if (flags <= 1)
203 len = *(unsigned short *)(va + 4);
204 else
205 len = *(unsigned int *)(va + 4);
206 if (pg_offset + len > kvm->arch.ram_psize)
207 return H_PARAMETER;
208 switch (flags) {
209 case 1: /* register VPA */
210 if (len < 640)
211 return H_PARAMETER;
212 tvcpu->arch.vpa = va;
213 init_vpa(vcpu, va);
214 break;
215 case 2: /* register DTL */
216 if (len < 48)
217 return H_PARAMETER;
218 if (!tvcpu->arch.vpa)
219 return H_RESOURCE;
220 len -= len % 48;
221 tvcpu->arch.dtl = va;
222 tvcpu->arch.dtl_end = va + len;
223 break;
224 case 3: /* register SLB shadow buffer */
225 if (len < 8)
226 return H_PARAMETER;
227 if (!tvcpu->arch.vpa)
228 return H_RESOURCE;
229 tvcpu->arch.slb_shadow = va;
230 len = (len - 16) / 16;
231 tvcpu->arch.slb_shadow = va;
232 break;
233 }
234 } else {
235 switch (flags) {
236 case 5: /* unregister VPA */
237 if (tvcpu->arch.slb_shadow || tvcpu->arch.dtl)
238 return H_RESOURCE;
239 tvcpu->arch.vpa = NULL;
240 break;
241 case 6: /* unregister DTL */
242 tvcpu->arch.dtl = NULL;
243 break;
244 case 7: /* unregister SLB shadow buffer */
245 tvcpu->arch.slb_shadow = NULL;
246 break;
247 }
248 }
249 return H_SUCCESS;
250}
251
252int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu)
253{
254 unsigned long req = kvmppc_get_gpr(vcpu, 3);
255 unsigned long target, ret = H_SUCCESS;
256 struct kvm_vcpu *tvcpu;
257
258 switch (req) {
259 case H_CEDE:
260 vcpu->arch.shregs.msr |= MSR_EE;
261 vcpu->arch.ceded = 1;
262 smp_mb();
263 if (!vcpu->arch.prodded)
264 kvmppc_vcpu_block(vcpu);
265 else
266 vcpu->arch.prodded = 0;
267 smp_mb();
268 vcpu->arch.ceded = 0;
269 break;
270 case H_PROD:
271 target = kvmppc_get_gpr(vcpu, 4);
272 tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);
273 if (!tvcpu) {
274 ret = H_PARAMETER;
275 break;
276 }
277 tvcpu->arch.prodded = 1;
278 smp_mb();
279 if (vcpu->arch.ceded) {
280 if (waitqueue_active(&vcpu->wq)) {
281 wake_up_interruptible(&vcpu->wq);
282 vcpu->stat.halt_wakeup++;
283 }
284 }
285 break;
286 case H_CONFER:
287 break;
288 case H_REGISTER_VPA:
289 ret = do_h_register_vpa(vcpu, kvmppc_get_gpr(vcpu, 4),
290 kvmppc_get_gpr(vcpu, 5),
291 kvmppc_get_gpr(vcpu, 6));
292 break;
293 default:
294 return RESUME_HOST;
295 }
296 kvmppc_set_gpr(vcpu, 3, ret);
297 vcpu->arch.hcall_needed = 0;
298 return RESUME_GUEST;
299}
300
301static int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu,
302 struct task_struct *tsk)
303{
304 int r = RESUME_HOST;
305
306 vcpu->stat.sum_exits++;
307
308 run->exit_reason = KVM_EXIT_UNKNOWN;
309 run->ready_for_interrupt_injection = 1;
310 switch (vcpu->arch.trap) {
311 /* We're good on these - the host merely wanted to get our attention */
312 case BOOK3S_INTERRUPT_HV_DECREMENTER:
313 vcpu->stat.dec_exits++;
314 r = RESUME_GUEST;
315 break;
316 case BOOK3S_INTERRUPT_EXTERNAL:
317 vcpu->stat.ext_intr_exits++;
318 r = RESUME_GUEST;
319 break;
320 case BOOK3S_INTERRUPT_PERFMON:
321 r = RESUME_GUEST;
322 break;
323 case BOOK3S_INTERRUPT_PROGRAM:
324 {
325 ulong flags;
326 /*
327 * Normally program interrupts are delivered directly
328 * to the guest by the hardware, but we can get here
329 * as a result of a hypervisor emulation interrupt
330 * (e40) getting turned into a 700 by BML RTAS.
331 */
332 flags = vcpu->arch.shregs.msr & 0x1f0000ull;
333 kvmppc_core_queue_program(vcpu, flags);
334 r = RESUME_GUEST;
335 break;
336 }
337 case BOOK3S_INTERRUPT_SYSCALL:
338 {
339 /* hcall - punt to userspace */
340 int i;
341
342 if (vcpu->arch.shregs.msr & MSR_PR) {
343 /* sc 1 from userspace - reflect to guest syscall */
344 kvmppc_book3s_queue_irqprio(vcpu, BOOK3S_INTERRUPT_SYSCALL);
345 r = RESUME_GUEST;
346 break;
347 }
348 run->papr_hcall.nr = kvmppc_get_gpr(vcpu, 3);
349 for (i = 0; i < 9; ++i)
350 run->papr_hcall.args[i] = kvmppc_get_gpr(vcpu, 4 + i);
351 run->exit_reason = KVM_EXIT_PAPR_HCALL;
352 vcpu->arch.hcall_needed = 1;
353 r = RESUME_HOST;
354 break;
355 }
356 /*
357 * We get these next two if the guest does a bad real-mode access,
358 * as we have enabled VRMA (virtualized real mode area) mode in the
359 * LPCR. We just generate an appropriate DSI/ISI to the guest.
360 */
361 case BOOK3S_INTERRUPT_H_DATA_STORAGE:
362 vcpu->arch.shregs.dsisr = vcpu->arch.fault_dsisr;
363 vcpu->arch.shregs.dar = vcpu->arch.fault_dar;
364 kvmppc_inject_interrupt(vcpu, BOOK3S_INTERRUPT_DATA_STORAGE, 0);
365 r = RESUME_GUEST;
366 break;
367 case BOOK3S_INTERRUPT_H_INST_STORAGE:
368 kvmppc_inject_interrupt(vcpu, BOOK3S_INTERRUPT_INST_STORAGE,
369 0x08000000);
370 r = RESUME_GUEST;
371 break;
372 /*
373 * This occurs if the guest executes an illegal instruction.
374 * We just generate a program interrupt to the guest, since
375 * we don't emulate any guest instructions at this stage.
376 */
377 case BOOK3S_INTERRUPT_H_EMUL_ASSIST:
378 kvmppc_core_queue_program(vcpu, 0x80000);
379 r = RESUME_GUEST;
380 break;
381 default:
382 kvmppc_dump_regs(vcpu);
383 printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n",
384 vcpu->arch.trap, kvmppc_get_pc(vcpu),
385 vcpu->arch.shregs.msr);
386 r = RESUME_HOST;
387 BUG();
388 break;
389 }
390
391
392 if (!(r & RESUME_HOST)) {
393 /* To avoid clobbering exit_reason, only check for signals if
394 * we aren't already exiting to userspace for some other
395 * reason. */
396 if (signal_pending(tsk)) {
397 vcpu->stat.signal_exits++;
398 run->exit_reason = KVM_EXIT_INTR;
399 r = -EINTR;
400 } else {
401 kvmppc_core_deliver_interrupts(vcpu);
402 }
403 }
404
405 return r;
406}
407
408int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
409 struct kvm_sregs *sregs)
410{
411 int i;
412
413 sregs->pvr = vcpu->arch.pvr;
414
415 memset(sregs, 0, sizeof(struct kvm_sregs));
416 for (i = 0; i < vcpu->arch.slb_max; i++) {
417 sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige;
418 sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv;
419 }
420
421 return 0;
422}
423
424int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
425 struct kvm_sregs *sregs)
426{
427 int i, j;
428
429 kvmppc_set_pvr(vcpu, sregs->pvr);
430
431 j = 0;
432 for (i = 0; i < vcpu->arch.slb_nr; i++) {
433 if (sregs->u.s.ppc64.slb[i].slbe & SLB_ESID_V) {
434 vcpu->arch.slb[j].orige = sregs->u.s.ppc64.slb[i].slbe;
435 vcpu->arch.slb[j].origv = sregs->u.s.ppc64.slb[i].slbv;
436 ++j;
437 }
438 }
439 vcpu->arch.slb_max = j;
440
441 return 0;
442}
443
444int kvmppc_core_check_processor_compat(void)
445{
446 if (cpu_has_feature(CPU_FTR_HVMODE))
447 return 0;
448 return -EIO;
449}
450
451struct kvm_vcpu *kvmppc_core_vcpu_create(struct kvm *kvm, unsigned int id)
452{
453 struct kvm_vcpu *vcpu;
454 int err = -EINVAL;
455 int core;
456 struct kvmppc_vcore *vcore;
457
458 core = id / threads_per_core;
459 if (core >= KVM_MAX_VCORES)
460 goto out;
461
462 err = -ENOMEM;
463 vcpu = kzalloc(sizeof(struct kvm_vcpu), GFP_KERNEL);
464 if (!vcpu)
465 goto out;
466
467 err = kvm_vcpu_init(vcpu, kvm, id);
468 if (err)
469 goto free_vcpu;
470
471 vcpu->arch.shared = &vcpu->arch.shregs;
472 vcpu->arch.last_cpu = -1;
473 vcpu->arch.mmcr[0] = MMCR0_FC;
474 vcpu->arch.ctrl = CTRL_RUNLATCH;
475 /* default to host PVR, since we can't spoof it */
476 vcpu->arch.pvr = mfspr(SPRN_PVR);
477 kvmppc_set_pvr(vcpu, vcpu->arch.pvr);
478
479 kvmppc_mmu_book3s_hv_init(vcpu);
480
481 /*
482 * Some vcpus may start out in stopped state. If we initialize
483 * them to busy-in-host state they will stop other vcpus in the
484 * vcore from running. Instead we initialize them to blocked
485 * state, effectively considering them to be stopped until we
486 * see the first run ioctl for them.
487 */
488 vcpu->arch.state = KVMPPC_VCPU_BLOCKED;
489
490 init_waitqueue_head(&vcpu->arch.cpu_run);
491
492 mutex_lock(&kvm->lock);
493 vcore = kvm->arch.vcores[core];
494 if (!vcore) {
495 vcore = kzalloc(sizeof(struct kvmppc_vcore), GFP_KERNEL);
496 if (vcore) {
497 INIT_LIST_HEAD(&vcore->runnable_threads);
498 spin_lock_init(&vcore->lock);
499 }
500 kvm->arch.vcores[core] = vcore;
501 }
502 mutex_unlock(&kvm->lock);
503
504 if (!vcore)
505 goto free_vcpu;
506
507 spin_lock(&vcore->lock);
508 ++vcore->num_threads;
509 ++vcore->n_blocked;
510 spin_unlock(&vcore->lock);
511 vcpu->arch.vcore = vcore;
512
513 return vcpu;
514
515free_vcpu:
516 kfree(vcpu);
517out:
518 return ERR_PTR(err);
519}
520
521void kvmppc_core_vcpu_free(struct kvm_vcpu *vcpu)
522{
523 kvm_vcpu_uninit(vcpu);
524 kfree(vcpu);
525}
526
527static void kvmppc_vcpu_blocked(struct kvm_vcpu *vcpu)
528{
529 struct kvmppc_vcore *vc = vcpu->arch.vcore;
530
531 spin_lock(&vc->lock);
532 vcpu->arch.state = KVMPPC_VCPU_BLOCKED;
533 ++vc->n_blocked;
534 if (vc->n_runnable > 0 &&
535 vc->n_runnable + vc->n_blocked == vc->num_threads) {
536 vcpu = list_first_entry(&vc->runnable_threads, struct kvm_vcpu,
537 arch.run_list);
538 wake_up(&vcpu->arch.cpu_run);
539 }
540 spin_unlock(&vc->lock);
541}
542
543static void kvmppc_vcpu_unblocked(struct kvm_vcpu *vcpu)
544{
545 struct kvmppc_vcore *vc = vcpu->arch.vcore;
546
547 spin_lock(&vc->lock);
548 vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
549 --vc->n_blocked;
550 spin_unlock(&vc->lock);
551}
552
553extern int __kvmppc_vcore_entry(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu);
554extern void xics_wake_cpu(int cpu);
555
556static void kvmppc_remove_runnable(struct kvmppc_vcore *vc,
557 struct kvm_vcpu *vcpu)
558{
559 struct kvm_vcpu *v;
560
561 if (vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
562 return;
563 vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
564 --vc->n_runnable;
565 /* decrement the physical thread id of each following vcpu */
566 v = vcpu;
567 list_for_each_entry_continue(v, &vc->runnable_threads, arch.run_list)
568 --v->arch.ptid;
569 list_del(&vcpu->arch.run_list);
570}
571
572static void kvmppc_start_thread(struct kvm_vcpu *vcpu)
573{
574 int cpu;
575 struct paca_struct *tpaca;
576 struct kvmppc_vcore *vc = vcpu->arch.vcore;
577
578 cpu = vc->pcpu + vcpu->arch.ptid;
579 tpaca = &paca[cpu];
580 tpaca->kvm_hstate.kvm_vcpu = vcpu;
581 tpaca->kvm_hstate.kvm_vcore = vc;
582 smp_wmb();
583#ifdef CONFIG_PPC_ICP_NATIVE
584 if (vcpu->arch.ptid) {
585 tpaca->cpu_start = 0x80;
586 tpaca->kvm_hstate.in_guest = KVM_GUEST_MODE_GUEST;
587 wmb();
588 xics_wake_cpu(cpu);
589 ++vc->n_woken;
590 }
591#endif
592}
593
594static void kvmppc_wait_for_nap(struct kvmppc_vcore *vc)
595{
596 int i;
597
598 HMT_low();
599 i = 0;
600 while (vc->nap_count < vc->n_woken) {
601 if (++i >= 1000000) {
602 pr_err("kvmppc_wait_for_nap timeout %d %d\n",
603 vc->nap_count, vc->n_woken);
604 break;
605 }
606 cpu_relax();
607 }
608 HMT_medium();
609}
610
611/*
612 * Check that we are on thread 0 and that any other threads in
613 * this core are off-line.
614 */
615static int on_primary_thread(void)
616{
617 int cpu = smp_processor_id();
618 int thr = cpu_thread_in_core(cpu);
619
620 if (thr)
621 return 0;
622 while (++thr < threads_per_core)
623 if (cpu_online(cpu + thr))
624 return 0;
625 return 1;
626}
627
628/*
629 * Run a set of guest threads on a physical core.
630 * Called with vc->lock held.
631 */
632static int kvmppc_run_core(struct kvmppc_vcore *vc)
633{
634 struct kvm_vcpu *vcpu, *vnext;
635 long ret;
636 u64 now;
637
638 /* don't start if any threads have a signal pending */
639 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list)
640 if (signal_pending(vcpu->arch.run_task))
641 return 0;
642
643 /*
644 * Make sure we are running on thread 0, and that
645 * secondary threads are offline.
646 * XXX we should also block attempts to bring any
647 * secondary threads online.
648 */
649 if (threads_per_core > 1 && !on_primary_thread()) {
650 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list)
651 vcpu->arch.ret = -EBUSY;
652 goto out;
653 }
654
655 vc->n_woken = 0;
656 vc->nap_count = 0;
657 vc->entry_exit_count = 0;
658 vc->vcore_running = 1;
659 vc->in_guest = 0;
660 vc->pcpu = smp_processor_id();
661 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list)
662 kvmppc_start_thread(vcpu);
663 vcpu = list_first_entry(&vc->runnable_threads, struct kvm_vcpu,
664 arch.run_list);
665
666 spin_unlock(&vc->lock);
667
668 preempt_disable();
669 kvm_guest_enter();
670 __kvmppc_vcore_entry(NULL, vcpu);
671
672 /* wait for secondary threads to finish writing their state to memory */
673 spin_lock(&vc->lock);
674 if (vc->nap_count < vc->n_woken)
675 kvmppc_wait_for_nap(vc);
676 /* prevent other vcpu threads from doing kvmppc_start_thread() now */
677 vc->vcore_running = 2;
678 spin_unlock(&vc->lock);
679
680 /* make sure updates to secondary vcpu structs are visible now */
681 smp_mb();
682 kvm_guest_exit();
683
684 preempt_enable();
685 kvm_resched(vcpu);
686
687 now = get_tb();
688 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) {
689 /* cancel pending dec exception if dec is positive */
690 if (now < vcpu->arch.dec_expires &&
691 kvmppc_core_pending_dec(vcpu))
692 kvmppc_core_dequeue_dec(vcpu);
693 if (!vcpu->arch.trap) {
694 if (signal_pending(vcpu->arch.run_task)) {
695 vcpu->arch.kvm_run->exit_reason = KVM_EXIT_INTR;
696 vcpu->arch.ret = -EINTR;
697 }
698 continue; /* didn't get to run */
699 }
700 ret = kvmppc_handle_exit(vcpu->arch.kvm_run, vcpu,
701 vcpu->arch.run_task);
702 vcpu->arch.ret = ret;
703 vcpu->arch.trap = 0;
704 }
705
706 spin_lock(&vc->lock);
707 out:
708 vc->vcore_running = 0;
709 list_for_each_entry_safe(vcpu, vnext, &vc->runnable_threads,
710 arch.run_list) {
711 if (vcpu->arch.ret != RESUME_GUEST) {
712 kvmppc_remove_runnable(vc, vcpu);
713 wake_up(&vcpu->arch.cpu_run);
714 }
715 }
716
717 return 1;
718}
719
720static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
721{
722 int ptid;
723 int wait_state;
724 struct kvmppc_vcore *vc;
725 DEFINE_WAIT(wait);
726
727 /* No need to go into the guest when all we do is going out */
728 if (signal_pending(current)) {
729 kvm_run->exit_reason = KVM_EXIT_INTR;
730 return -EINTR;
731 }
732
733 /* On PPC970, check that we have an RMA region */
734 if (!vcpu->kvm->arch.rma && cpu_has_feature(CPU_FTR_ARCH_201))
735 return -EPERM;
736
737 kvm_run->exit_reason = 0;
738 vcpu->arch.ret = RESUME_GUEST;
739 vcpu->arch.trap = 0;
740
741 flush_fp_to_thread(current);
742 flush_altivec_to_thread(current);
743 flush_vsx_to_thread(current);
744
745 /*
746 * Synchronize with other threads in this virtual core
747 */
748 vc = vcpu->arch.vcore;
749 spin_lock(&vc->lock);
750 /* This happens the first time this is called for a vcpu */
751 if (vcpu->arch.state == KVMPPC_VCPU_BLOCKED)
752 --vc->n_blocked;
753 vcpu->arch.state = KVMPPC_VCPU_RUNNABLE;
754 ptid = vc->n_runnable;
755 vcpu->arch.run_task = current;
756 vcpu->arch.kvm_run = kvm_run;
757 vcpu->arch.ptid = ptid;
758 list_add_tail(&vcpu->arch.run_list, &vc->runnable_threads);
759 ++vc->n_runnable;
760
761 wait_state = TASK_INTERRUPTIBLE;
762 while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) {
763 if (signal_pending(current)) {
764 if (!vc->vcore_running) {
765 kvm_run->exit_reason = KVM_EXIT_INTR;
766 vcpu->arch.ret = -EINTR;
767 break;
768 }
769 /* have to wait for vcore to stop executing guest */
770 wait_state = TASK_UNINTERRUPTIBLE;
771 smp_send_reschedule(vc->pcpu);
772 }
773
774 if (!vc->vcore_running &&
775 vc->n_runnable + vc->n_blocked == vc->num_threads) {
776 /* we can run now */
777 if (kvmppc_run_core(vc))
778 continue;
779 }
780
781 if (vc->vcore_running == 1 && VCORE_EXIT_COUNT(vc) == 0)
782 kvmppc_start_thread(vcpu);
783
784 /* wait for other threads to come in, or wait for vcore */
785 prepare_to_wait(&vcpu->arch.cpu_run, &wait, wait_state);
786 spin_unlock(&vc->lock);
787 schedule();
788 finish_wait(&vcpu->arch.cpu_run, &wait);
789 spin_lock(&vc->lock);
790 }
791
792 if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE)
793 kvmppc_remove_runnable(vc, vcpu);
794 spin_unlock(&vc->lock);
795
796 return vcpu->arch.ret;
797}
798
799int kvmppc_vcpu_run(struct kvm_run *run, struct kvm_vcpu *vcpu)
800{
801 int r;
802
803 do {
804 r = kvmppc_run_vcpu(run, vcpu);
805
806 if (run->exit_reason == KVM_EXIT_PAPR_HCALL &&
807 !(vcpu->arch.shregs.msr & MSR_PR)) {
808 r = kvmppc_pseries_do_hcall(vcpu);
809 kvmppc_core_deliver_interrupts(vcpu);
810 }
811 } while (r == RESUME_GUEST);
812 return r;
813}
814
815static long kvmppc_stt_npages(unsigned long window_size)
816{
817 return ALIGN((window_size >> SPAPR_TCE_SHIFT)
818 * sizeof(u64), PAGE_SIZE) / PAGE_SIZE;
819}
820
821static void release_spapr_tce_table(struct kvmppc_spapr_tce_table *stt)
822{
823 struct kvm *kvm = stt->kvm;
824 int i;
825
826 mutex_lock(&kvm->lock);
827 list_del(&stt->list);
828 for (i = 0; i < kvmppc_stt_npages(stt->window_size); i++)
829 __free_page(stt->pages[i]);
830 kfree(stt);
831 mutex_unlock(&kvm->lock);
832
833 kvm_put_kvm(kvm);
834}
835
836static int kvm_spapr_tce_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
837{
838 struct kvmppc_spapr_tce_table *stt = vma->vm_file->private_data;
839 struct page *page;
840
841 if (vmf->pgoff >= kvmppc_stt_npages(stt->window_size))
842 return VM_FAULT_SIGBUS;
843
844 page = stt->pages[vmf->pgoff];
845 get_page(page);
846 vmf->page = page;
847 return 0;
848}
849
850static const struct vm_operations_struct kvm_spapr_tce_vm_ops = {
851 .fault = kvm_spapr_tce_fault,
852};
853
854static int kvm_spapr_tce_mmap(struct file *file, struct vm_area_struct *vma)
855{
856 vma->vm_ops = &kvm_spapr_tce_vm_ops;
857 return 0;
858}
859
860static int kvm_spapr_tce_release(struct inode *inode, struct file *filp)
861{
862 struct kvmppc_spapr_tce_table *stt = filp->private_data;
863
864 release_spapr_tce_table(stt);
865 return 0;
866}
867
868static struct file_operations kvm_spapr_tce_fops = {
869 .mmap = kvm_spapr_tce_mmap,
870 .release = kvm_spapr_tce_release,
871};
872
873long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
874 struct kvm_create_spapr_tce *args)
875{
876 struct kvmppc_spapr_tce_table *stt = NULL;
877 long npages;
878 int ret = -ENOMEM;
879 int i;
880
881 /* Check this LIOBN hasn't been previously allocated */
882 list_for_each_entry(stt, &kvm->arch.spapr_tce_tables, list) {
883 if (stt->liobn == args->liobn)
884 return -EBUSY;
885 }
886
887 npages = kvmppc_stt_npages(args->window_size);
888
889 stt = kzalloc(sizeof(*stt) + npages* sizeof(struct page *),
890 GFP_KERNEL);
891 if (!stt)
892 goto fail;
893
894 stt->liobn = args->liobn;
895 stt->window_size = args->window_size;
896 stt->kvm = kvm;
897
898 for (i = 0; i < npages; i++) {
899 stt->pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO);
900 if (!stt->pages[i])
901 goto fail;
902 }
903
904 kvm_get_kvm(kvm);
905
906 mutex_lock(&kvm->lock);
907 list_add(&stt->list, &kvm->arch.spapr_tce_tables);
908
909 mutex_unlock(&kvm->lock);
910
911 return anon_inode_getfd("kvm-spapr-tce", &kvm_spapr_tce_fops,
912 stt, O_RDWR);
913
914fail:
915 if (stt) {
916 for (i = 0; i < npages; i++)
917 if (stt->pages[i])
918 __free_page(stt->pages[i]);
919
920 kfree(stt);
921 }
922 return ret;
923}
924
925/* Work out RMLS (real mode limit selector) field value for a given RMA size.
926 Assumes POWER7 or PPC970. */
927static inline int lpcr_rmls(unsigned long rma_size)
928{
929 switch (rma_size) {
930 case 32ul << 20: /* 32 MB */
931 if (cpu_has_feature(CPU_FTR_ARCH_206))
932 return 8; /* only supported on POWER7 */
933 return -1;
934 case 64ul << 20: /* 64 MB */
935 return 3;
936 case 128ul << 20: /* 128 MB */
937 return 7;
938 case 256ul << 20: /* 256 MB */
939 return 4;
940 case 1ul << 30: /* 1 GB */
941 return 2;
942 case 16ul << 30: /* 16 GB */
943 return 1;
944 case 256ul << 30: /* 256 GB */
945 return 0;
946 default:
947 return -1;
948 }
949}
950
951static int kvm_rma_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
952{
953 struct kvmppc_rma_info *ri = vma->vm_file->private_data;
954 struct page *page;
955
956 if (vmf->pgoff >= ri->npages)
957 return VM_FAULT_SIGBUS;
958
959 page = pfn_to_page(ri->base_pfn + vmf->pgoff);
960 get_page(page);
961 vmf->page = page;
962 return 0;
963}
964
965static const struct vm_operations_struct kvm_rma_vm_ops = {
966 .fault = kvm_rma_fault,
967};
968
969static int kvm_rma_mmap(struct file *file, struct vm_area_struct *vma)
970{
971 vma->vm_flags |= VM_RESERVED;
972 vma->vm_ops = &kvm_rma_vm_ops;
973 return 0;
974}
975
976static int kvm_rma_release(struct inode *inode, struct file *filp)
977{
978 struct kvmppc_rma_info *ri = filp->private_data;
979
980 kvm_release_rma(ri);
981 return 0;
982}
983
984static struct file_operations kvm_rma_fops = {
985 .mmap = kvm_rma_mmap,
986 .release = kvm_rma_release,
987};
988
989long kvm_vm_ioctl_allocate_rma(struct kvm *kvm, struct kvm_allocate_rma *ret)
990{
991 struct kvmppc_rma_info *ri;
992 long fd;
993
994 ri = kvm_alloc_rma();
995 if (!ri)
996 return -ENOMEM;
997
998 fd = anon_inode_getfd("kvm-rma", &kvm_rma_fops, ri, O_RDWR);
999 if (fd < 0)
1000 kvm_release_rma(ri);
1001
1002 ret->rma_size = ri->npages << PAGE_SHIFT;
1003 return fd;
1004}
1005
1006static struct page *hva_to_page(unsigned long addr)
1007{
1008 struct page *page[1];
1009 int npages;
1010
1011 might_sleep();
1012
1013 npages = get_user_pages_fast(addr, 1, 1, page);
1014
1015 if (unlikely(npages != 1))
1016 return 0;
1017
1018 return page[0];
1019}
1020
1021int kvmppc_core_prepare_memory_region(struct kvm *kvm,
1022 struct kvm_userspace_memory_region *mem)
1023{
1024 unsigned long psize, porder;
1025 unsigned long i, npages, totalpages;
1026 unsigned long pg_ix;
1027 struct kvmppc_pginfo *pginfo;
1028 unsigned long hva;
1029 struct kvmppc_rma_info *ri = NULL;
1030 struct page *page;
1031
1032 /* For now, only allow 16MB pages */
1033 porder = LARGE_PAGE_ORDER;
1034 psize = 1ul << porder;
1035 if ((mem->memory_size & (psize - 1)) ||
1036 (mem->guest_phys_addr & (psize - 1))) {
1037 pr_err("bad memory_size=%llx @ %llx\n",
1038 mem->memory_size, mem->guest_phys_addr);
1039 return -EINVAL;
1040 }
1041
1042 npages = mem->memory_size >> porder;
1043 totalpages = (mem->guest_phys_addr + mem->memory_size) >> porder;
1044
1045 /* More memory than we have space to track? */
1046 if (totalpages > (1ul << (MAX_MEM_ORDER - LARGE_PAGE_ORDER)))
1047 return -EINVAL;
1048
1049 /* Do we already have an RMA registered? */
1050 if (mem->guest_phys_addr == 0 && kvm->arch.rma)
1051 return -EINVAL;
1052
1053 if (totalpages > kvm->arch.ram_npages)
1054 kvm->arch.ram_npages = totalpages;
1055
1056 /* Is this one of our preallocated RMAs? */
1057 if (mem->guest_phys_addr == 0) {
1058 struct vm_area_struct *vma;
1059
1060 down_read(&current->mm->mmap_sem);
1061 vma = find_vma(current->mm, mem->userspace_addr);
1062 if (vma && vma->vm_file &&
1063 vma->vm_file->f_op == &kvm_rma_fops &&
1064 mem->userspace_addr == vma->vm_start)
1065 ri = vma->vm_file->private_data;
1066 up_read(&current->mm->mmap_sem);
1067 if (!ri && cpu_has_feature(CPU_FTR_ARCH_201)) {
1068 pr_err("CPU requires an RMO\n");
1069 return -EINVAL;
1070 }
1071 }
1072
1073 if (ri) {
1074 unsigned long rma_size;
1075 unsigned long lpcr;
1076 long rmls;
1077
1078 rma_size = ri->npages << PAGE_SHIFT;
1079 if (rma_size > mem->memory_size)
1080 rma_size = mem->memory_size;
1081 rmls = lpcr_rmls(rma_size);
1082 if (rmls < 0) {
1083 pr_err("Can't use RMA of 0x%lx bytes\n", rma_size);
1084 return -EINVAL;
1085 }
1086 atomic_inc(&ri->use_count);
1087 kvm->arch.rma = ri;
1088 kvm->arch.n_rma_pages = rma_size >> porder;
1089
1090 /* Update LPCR and RMOR */
1091 lpcr = kvm->arch.lpcr;
1092 if (cpu_has_feature(CPU_FTR_ARCH_201)) {
1093 /* PPC970; insert RMLS value (split field) in HID4 */
1094 lpcr &= ~((1ul << HID4_RMLS0_SH) |
1095 (3ul << HID4_RMLS2_SH));
1096 lpcr |= ((rmls >> 2) << HID4_RMLS0_SH) |
1097 ((rmls & 3) << HID4_RMLS2_SH);
1098 /* RMOR is also in HID4 */
1099 lpcr |= ((ri->base_pfn >> (26 - PAGE_SHIFT)) & 0xffff)
1100 << HID4_RMOR_SH;
1101 } else {
1102 /* POWER7 */
1103 lpcr &= ~(LPCR_VPM0 | LPCR_VRMA_L);
1104 lpcr |= rmls << LPCR_RMLS_SH;
1105 kvm->arch.rmor = kvm->arch.rma->base_pfn << PAGE_SHIFT;
1106 }
1107 kvm->arch.lpcr = lpcr;
1108 pr_info("Using RMO at %lx size %lx (LPCR = %lx)\n",
1109 ri->base_pfn << PAGE_SHIFT, rma_size, lpcr);
1110 }
1111
1112 pg_ix = mem->guest_phys_addr >> porder;
1113 pginfo = kvm->arch.ram_pginfo + pg_ix;
1114 for (i = 0; i < npages; ++i, ++pg_ix) {
1115 if (ri && pg_ix < kvm->arch.n_rma_pages) {
1116 pginfo[i].pfn = ri->base_pfn +
1117 (pg_ix << (porder - PAGE_SHIFT));
1118 continue;
1119 }
1120 hva = mem->userspace_addr + (i << porder);
1121 page = hva_to_page(hva);
1122 if (!page) {
1123 pr_err("oops, no pfn for hva %lx\n", hva);
1124 goto err;
1125 }
1126 /* Check it's a 16MB page */
1127 if (!PageHead(page) ||
1128 compound_order(page) != (LARGE_PAGE_ORDER - PAGE_SHIFT)) {
1129 pr_err("page at %lx isn't 16MB (o=%d)\n",
1130 hva, compound_order(page));
1131 goto err;
1132 }
1133 pginfo[i].pfn = page_to_pfn(page);
1134 }
1135
1136 return 0;
1137
1138 err:
1139 return -EINVAL;
1140}
1141
1142void kvmppc_core_commit_memory_region(struct kvm *kvm,
1143 struct kvm_userspace_memory_region *mem)
1144{
1145 if (mem->guest_phys_addr == 0 && mem->memory_size != 0 &&
1146 !kvm->arch.rma)
1147 kvmppc_map_vrma(kvm, mem);
1148}
1149
1150int kvmppc_core_init_vm(struct kvm *kvm)
1151{
1152 long r;
1153 unsigned long npages = 1ul << (MAX_MEM_ORDER - LARGE_PAGE_ORDER);
1154 long err = -ENOMEM;
1155 unsigned long lpcr;
1156
1157 /* Allocate hashed page table */
1158 r = kvmppc_alloc_hpt(kvm);
1159 if (r)
1160 return r;
1161
1162 INIT_LIST_HEAD(&kvm->arch.spapr_tce_tables);
1163
1164 kvm->arch.ram_pginfo = kzalloc(npages * sizeof(struct kvmppc_pginfo),
1165 GFP_KERNEL);
1166 if (!kvm->arch.ram_pginfo) {
1167 pr_err("kvmppc_core_init_vm: couldn't alloc %lu bytes\n",
1168 npages * sizeof(struct kvmppc_pginfo));
1169 goto out_free;
1170 }
1171
1172 kvm->arch.ram_npages = 0;
1173 kvm->arch.ram_psize = 1ul << LARGE_PAGE_ORDER;
1174 kvm->arch.ram_porder = LARGE_PAGE_ORDER;
1175 kvm->arch.rma = NULL;
1176 kvm->arch.n_rma_pages = 0;
1177
1178 kvm->arch.host_sdr1 = mfspr(SPRN_SDR1);
1179
1180 if (cpu_has_feature(CPU_FTR_ARCH_201)) {
1181 /* PPC970; HID4 is effectively the LPCR */
1182 unsigned long lpid = kvm->arch.lpid;
1183 kvm->arch.host_lpid = 0;
1184 kvm->arch.host_lpcr = lpcr = mfspr(SPRN_HID4);
1185 lpcr &= ~((3 << HID4_LPID1_SH) | (0xful << HID4_LPID5_SH));
1186 lpcr |= ((lpid >> 4) << HID4_LPID1_SH) |
1187 ((lpid & 0xf) << HID4_LPID5_SH);
1188 } else {
1189 /* POWER7; init LPCR for virtual RMA mode */
1190 kvm->arch.host_lpid = mfspr(SPRN_LPID);
1191 kvm->arch.host_lpcr = lpcr = mfspr(SPRN_LPCR);
1192 lpcr &= LPCR_PECE | LPCR_LPES;
1193 lpcr |= (4UL << LPCR_DPFD_SH) | LPCR_HDICE |
1194 LPCR_VPM0 | LPCR_VRMA_L;
1195 }
1196 kvm->arch.lpcr = lpcr;
1197
1198 return 0;
1199
1200 out_free:
1201 kvmppc_free_hpt(kvm);
1202 return err;
1203}
1204
1205void kvmppc_core_destroy_vm(struct kvm *kvm)
1206{
1207 struct kvmppc_pginfo *pginfo;
1208 unsigned long i;
1209
1210 if (kvm->arch.ram_pginfo) {
1211 pginfo = kvm->arch.ram_pginfo;
1212 kvm->arch.ram_pginfo = NULL;
1213 for (i = kvm->arch.n_rma_pages; i < kvm->arch.ram_npages; ++i)
1214 if (pginfo[i].pfn)
1215 put_page(pfn_to_page(pginfo[i].pfn));
1216 kfree(pginfo);
1217 }
1218 if (kvm->arch.rma) {
1219 kvm_release_rma(kvm->arch.rma);
1220 kvm->arch.rma = NULL;
1221 }
1222
1223 kvmppc_free_hpt(kvm);
1224 WARN_ON(!list_empty(&kvm->arch.spapr_tce_tables));
1225}
1226
1227/* These are stubs for now */
1228void kvmppc_mmu_pte_pflush(struct kvm_vcpu *vcpu, ulong pa_start, ulong pa_end)
1229{
1230}
1231
1232/* We don't need to emulate any privileged instructions or dcbz */
1233int kvmppc_core_emulate_op(struct kvm_run *run, struct kvm_vcpu *vcpu,
1234 unsigned int inst, int *advance)
1235{
1236 return EMULATE_FAIL;
1237}
1238
1239int kvmppc_core_emulate_mtspr(struct kvm_vcpu *vcpu, int sprn, int rs)
1240{
1241 return EMULATE_FAIL;
1242}
1243
1244int kvmppc_core_emulate_mfspr(struct kvm_vcpu *vcpu, int sprn, int rt)
1245{
1246 return EMULATE_FAIL;
1247}
1248
1249static int kvmppc_book3s_hv_init(void)
1250{
1251 int r;
1252
1253 r = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
1254
1255 if (r)
1256 return r;
1257
1258 r = kvmppc_mmu_hv_init();
1259
1260 return r;
1261}
1262
1263static void kvmppc_book3s_hv_exit(void)
1264{
1265 kvm_exit();
1266}
1267
1268module_init(kvmppc_book3s_hv_init);
1269module_exit(kvmppc_book3s_hv_exit);
diff --git a/arch/powerpc/kvm/book3s_hv_builtin.c b/arch/powerpc/kvm/book3s_hv_builtin.c
new file mode 100644
index 000000000000..d43120355eec
--- /dev/null
+++ b/arch/powerpc/kvm/book3s_hv_builtin.c
@@ -0,0 +1,155 @@
1/*
2 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License, version 2, as
6 * published by the Free Software Foundation.
7 */
8
9#include <linux/kvm_host.h>
10#include <linux/preempt.h>
11#include <linux/sched.h>
12#include <linux/spinlock.h>
13#include <linux/bootmem.h>
14#include <linux/init.h>
15
16#include <asm/cputable.h>
17#include <asm/kvm_ppc.h>
18#include <asm/kvm_book3s.h>
19
20/*
21 * This maintains a list of RMAs (real mode areas) for KVM guests to use.
22 * Each RMA has to be physically contiguous and of a size that the
23 * hardware supports. PPC970 and POWER7 support 64MB, 128MB and 256MB,
24 * and other larger sizes. Since we are unlikely to be allocate that
25 * much physically contiguous memory after the system is up and running,
26 * we preallocate a set of RMAs in early boot for KVM to use.
27 */
28static unsigned long kvm_rma_size = 64 << 20; /* 64MB */
29static unsigned long kvm_rma_count;
30
31static int __init early_parse_rma_size(char *p)
32{
33 if (!p)
34 return 1;
35
36 kvm_rma_size = memparse(p, &p);
37
38 return 0;
39}
40early_param("kvm_rma_size", early_parse_rma_size);
41
42static int __init early_parse_rma_count(char *p)
43{
44 if (!p)
45 return 1;
46
47 kvm_rma_count = simple_strtoul(p, NULL, 0);
48
49 return 0;
50}
51early_param("kvm_rma_count", early_parse_rma_count);
52
53static struct kvmppc_rma_info *rma_info;
54static LIST_HEAD(free_rmas);
55static DEFINE_SPINLOCK(rma_lock);
56
57/* Work out RMLS (real mode limit selector) field value for a given RMA size.
58 Assumes POWER7 or PPC970. */
59static inline int lpcr_rmls(unsigned long rma_size)
60{
61 switch (rma_size) {
62 case 32ul << 20: /* 32 MB */
63 if (cpu_has_feature(CPU_FTR_ARCH_206))
64 return 8; /* only supported on POWER7 */
65 return -1;
66 case 64ul << 20: /* 64 MB */
67 return 3;
68 case 128ul << 20: /* 128 MB */
69 return 7;
70 case 256ul << 20: /* 256 MB */
71 return 4;
72 case 1ul << 30: /* 1 GB */
73 return 2;
74 case 16ul << 30: /* 16 GB */
75 return 1;
76 case 256ul << 30: /* 256 GB */
77 return 0;
78 default:
79 return -1;
80 }
81}
82
83/*
84 * Called at boot time while the bootmem allocator is active,
85 * to allocate contiguous physical memory for the real memory
86 * areas for guests.
87 */
88void kvm_rma_init(void)
89{
90 unsigned long i;
91 unsigned long j, npages;
92 void *rma;
93 struct page *pg;
94
95 /* Only do this on PPC970 in HV mode */
96 if (!cpu_has_feature(CPU_FTR_HVMODE) ||
97 !cpu_has_feature(CPU_FTR_ARCH_201))
98 return;
99
100 if (!kvm_rma_size || !kvm_rma_count)
101 return;
102
103 /* Check that the requested size is one supported in hardware */
104 if (lpcr_rmls(kvm_rma_size) < 0) {
105 pr_err("RMA size of 0x%lx not supported\n", kvm_rma_size);
106 return;
107 }
108
109 npages = kvm_rma_size >> PAGE_SHIFT;
110 rma_info = alloc_bootmem(kvm_rma_count * sizeof(struct kvmppc_rma_info));
111 for (i = 0; i < kvm_rma_count; ++i) {
112 rma = alloc_bootmem_align(kvm_rma_size, kvm_rma_size);
113 pr_info("Allocated KVM RMA at %p (%ld MB)\n", rma,
114 kvm_rma_size >> 20);
115 rma_info[i].base_virt = rma;
116 rma_info[i].base_pfn = __pa(rma) >> PAGE_SHIFT;
117 rma_info[i].npages = npages;
118 list_add_tail(&rma_info[i].list, &free_rmas);
119 atomic_set(&rma_info[i].use_count, 0);
120
121 pg = pfn_to_page(rma_info[i].base_pfn);
122 for (j = 0; j < npages; ++j) {
123 atomic_inc(&pg->_count);
124 ++pg;
125 }
126 }
127}
128
129struct kvmppc_rma_info *kvm_alloc_rma(void)
130{
131 struct kvmppc_rma_info *ri;
132
133 ri = NULL;
134 spin_lock(&rma_lock);
135 if (!list_empty(&free_rmas)) {
136 ri = list_first_entry(&free_rmas, struct kvmppc_rma_info, list);
137 list_del(&ri->list);
138 atomic_inc(&ri->use_count);
139 }
140 spin_unlock(&rma_lock);
141 return ri;
142}
143EXPORT_SYMBOL_GPL(kvm_alloc_rma);
144
145void kvm_release_rma(struct kvmppc_rma_info *ri)
146{
147 if (atomic_dec_and_test(&ri->use_count)) {
148 spin_lock(&rma_lock);
149 list_add_tail(&ri->list, &free_rmas);
150 spin_unlock(&rma_lock);
151
152 }
153}
154EXPORT_SYMBOL_GPL(kvm_release_rma);
155
diff --git a/arch/powerpc/kvm/book3s_hv_interrupts.S b/arch/powerpc/kvm/book3s_hv_interrupts.S
new file mode 100644
index 000000000000..3f7b674dd4bf
--- /dev/null
+++ b/arch/powerpc/kvm/book3s_hv_interrupts.S
@@ -0,0 +1,166 @@
1/*
2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License, version 2, as
4 * published by the Free Software Foundation.
5 *
6 * This program is distributed in the hope that it will be useful,
7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9 * GNU General Public License for more details.
10 *
11 * You should have received a copy of the GNU General Public License
12 * along with this program; if not, write to the Free Software
13 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
14 *
15 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
16 *
17 * Derived from book3s_interrupts.S, which is:
18 * Copyright SUSE Linux Products GmbH 2009
19 *
20 * Authors: Alexander Graf <agraf@suse.de>
21 */
22
23#include <asm/ppc_asm.h>
24#include <asm/kvm_asm.h>
25#include <asm/reg.h>
26#include <asm/page.h>
27#include <asm/asm-offsets.h>
28#include <asm/exception-64s.h>
29#include <asm/ppc-opcode.h>
30
31/*****************************************************************************
32 * *
33 * Guest entry / exit code that is in kernel module memory (vmalloc) *
34 * *
35 ****************************************************************************/
36
37/* Registers:
38 * r4: vcpu pointer
39 */
40_GLOBAL(__kvmppc_vcore_entry)
41
42 /* Write correct stack frame */
43 mflr r0
44 std r0,PPC_LR_STKOFF(r1)
45
46 /* Save host state to the stack */
47 stdu r1, -SWITCH_FRAME_SIZE(r1)
48
49 /* Save non-volatile registers (r14 - r31) */
50 SAVE_NVGPRS(r1)
51
52 /* Save host DSCR */
53BEGIN_FTR_SECTION
54 mfspr r3, SPRN_DSCR
55 std r3, HSTATE_DSCR(r13)
56END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
57
58 /* Save host DABR */
59 mfspr r3, SPRN_DABR
60 std r3, HSTATE_DABR(r13)
61
62 /* Hard-disable interrupts */
63 mfmsr r10
64 std r10, HSTATE_HOST_MSR(r13)
65 rldicl r10,r10,48,1
66 rotldi r10,r10,16
67 mtmsrd r10,1
68
69 /* Save host PMU registers and load guest PMU registers */
70 /* R4 is live here (vcpu pointer) but not r3 or r5 */
71 li r3, 1
72 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */
73 mfspr r7, SPRN_MMCR0 /* save MMCR0 */
74 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable interrupts */
75 isync
76 ld r3, PACALPPACAPTR(r13) /* is the host using the PMU? */
77 lbz r5, LPPACA_PMCINUSE(r3)
78 cmpwi r5, 0
79 beq 31f /* skip if not */
80 mfspr r5, SPRN_MMCR1
81 mfspr r6, SPRN_MMCRA
82 std r7, HSTATE_MMCR(r13)
83 std r5, HSTATE_MMCR + 8(r13)
84 std r6, HSTATE_MMCR + 16(r13)
85 mfspr r3, SPRN_PMC1
86 mfspr r5, SPRN_PMC2
87 mfspr r6, SPRN_PMC3
88 mfspr r7, SPRN_PMC4
89 mfspr r8, SPRN_PMC5
90 mfspr r9, SPRN_PMC6
91BEGIN_FTR_SECTION
92 mfspr r10, SPRN_PMC7
93 mfspr r11, SPRN_PMC8
94END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
95 stw r3, HSTATE_PMC(r13)
96 stw r5, HSTATE_PMC + 4(r13)
97 stw r6, HSTATE_PMC + 8(r13)
98 stw r7, HSTATE_PMC + 12(r13)
99 stw r8, HSTATE_PMC + 16(r13)
100 stw r9, HSTATE_PMC + 20(r13)
101BEGIN_FTR_SECTION
102 stw r10, HSTATE_PMC + 24(r13)
103 stw r11, HSTATE_PMC + 28(r13)
104END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
10531:
106
107 /*
108 * Put whatever is in the decrementer into the
109 * hypervisor decrementer.
110 */
111 mfspr r8,SPRN_DEC
112 mftb r7
113 mtspr SPRN_HDEC,r8
114 extsw r8,r8
115 add r8,r8,r7
116 std r8,HSTATE_DECEXP(r13)
117
118 /*
119 * On PPC970, if the guest vcpu has an external interrupt pending,
120 * send ourselves an IPI so as to interrupt the guest once it
121 * enables interrupts. (It must have interrupts disabled,
122 * otherwise we would already have delivered the interrupt.)
123 */
124BEGIN_FTR_SECTION
125 ld r0, VCPU_PENDING_EXC(r4)
126 li r7, (1 << BOOK3S_IRQPRIO_EXTERNAL)
127 oris r7, r7, (1 << BOOK3S_IRQPRIO_EXTERNAL_LEVEL)@h
128 and. r0, r0, r7
129 beq 32f
130 mr r31, r4
131 lhz r3, PACAPACAINDEX(r13)
132 bl smp_send_reschedule
133 nop
134 mr r4, r31
13532:
136END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
137
138 /* Jump to partition switch code */
139 bl .kvmppc_hv_entry_trampoline
140 nop
141
142/*
143 * We return here in virtual mode after the guest exits
144 * with something that we can't handle in real mode.
145 * Interrupts are enabled again at this point.
146 */
147
148.global kvmppc_handler_highmem
149kvmppc_handler_highmem:
150
151 /*
152 * Register usage at this point:
153 *
154 * R1 = host R1
155 * R2 = host R2
156 * R12 = exit handler id
157 * R13 = PACA
158 */
159
160 /* Restore non-volatile host registers (r14 - r31) */
161 REST_NVGPRS(r1)
162
163 addi r1, r1, SWITCH_FRAME_SIZE
164 ld r0, PPC_LR_STKOFF(r1)
165 mtlr r0
166 blr
diff --git a/arch/powerpc/kvm/book3s_hv_rm_mmu.c b/arch/powerpc/kvm/book3s_hv_rm_mmu.c
new file mode 100644
index 000000000000..fcfe6b055558
--- /dev/null
+++ b/arch/powerpc/kvm/book3s_hv_rm_mmu.c
@@ -0,0 +1,370 @@
1/*
2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License, version 2, as
4 * published by the Free Software Foundation.
5 *
6 * Copyright 2010-2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
7 */
8
9#include <linux/types.h>
10#include <linux/string.h>
11#include <linux/kvm.h>
12#include <linux/kvm_host.h>
13#include <linux/hugetlb.h>
14
15#include <asm/tlbflush.h>
16#include <asm/kvm_ppc.h>
17#include <asm/kvm_book3s.h>
18#include <asm/mmu-hash64.h>
19#include <asm/hvcall.h>
20#include <asm/synch.h>
21#include <asm/ppc-opcode.h>
22
23/* For now use fixed-size 16MB page table */
24#define HPT_ORDER 24
25#define HPT_NPTEG (1ul << (HPT_ORDER - 7)) /* 128B per pteg */
26#define HPT_HASH_MASK (HPT_NPTEG - 1)
27
28#define HPTE_V_HVLOCK 0x40UL
29
30static inline long lock_hpte(unsigned long *hpte, unsigned long bits)
31{
32 unsigned long tmp, old;
33
34 asm volatile(" ldarx %0,0,%2\n"
35 " and. %1,%0,%3\n"
36 " bne 2f\n"
37 " ori %0,%0,%4\n"
38 " stdcx. %0,0,%2\n"
39 " beq+ 2f\n"
40 " li %1,%3\n"
41 "2: isync"
42 : "=&r" (tmp), "=&r" (old)
43 : "r" (hpte), "r" (bits), "i" (HPTE_V_HVLOCK)
44 : "cc", "memory");
45 return old == 0;
46}
47
48long kvmppc_h_enter(struct kvm_vcpu *vcpu, unsigned long flags,
49 long pte_index, unsigned long pteh, unsigned long ptel)
50{
51 unsigned long porder;
52 struct kvm *kvm = vcpu->kvm;
53 unsigned long i, lpn, pa;
54 unsigned long *hpte;
55
56 /* only handle 4k, 64k and 16M pages for now */
57 porder = 12;
58 if (pteh & HPTE_V_LARGE) {
59 if (cpu_has_feature(CPU_FTR_ARCH_206) &&
60 (ptel & 0xf000) == 0x1000) {
61 /* 64k page */
62 porder = 16;
63 } else if ((ptel & 0xff000) == 0) {
64 /* 16M page */
65 porder = 24;
66 /* lowest AVA bit must be 0 for 16M pages */
67 if (pteh & 0x80)
68 return H_PARAMETER;
69 } else
70 return H_PARAMETER;
71 }
72 lpn = (ptel & HPTE_R_RPN) >> kvm->arch.ram_porder;
73 if (lpn >= kvm->arch.ram_npages || porder > kvm->arch.ram_porder)
74 return H_PARAMETER;
75 pa = kvm->arch.ram_pginfo[lpn].pfn << PAGE_SHIFT;
76 if (!pa)
77 return H_PARAMETER;
78 /* Check WIMG */
79 if ((ptel & HPTE_R_WIMG) != HPTE_R_M &&
80 (ptel & HPTE_R_WIMG) != (HPTE_R_W | HPTE_R_I | HPTE_R_M))
81 return H_PARAMETER;
82 pteh &= ~0x60UL;
83 ptel &= ~(HPTE_R_PP0 - kvm->arch.ram_psize);
84 ptel |= pa;
85 if (pte_index >= (HPT_NPTEG << 3))
86 return H_PARAMETER;
87 if (likely((flags & H_EXACT) == 0)) {
88 pte_index &= ~7UL;
89 hpte = (unsigned long *)(kvm->arch.hpt_virt + (pte_index << 4));
90 for (i = 0; ; ++i) {
91 if (i == 8)
92 return H_PTEG_FULL;
93 if ((*hpte & HPTE_V_VALID) == 0 &&
94 lock_hpte(hpte, HPTE_V_HVLOCK | HPTE_V_VALID))
95 break;
96 hpte += 2;
97 }
98 } else {
99 i = 0;
100 hpte = (unsigned long *)(kvm->arch.hpt_virt + (pte_index << 4));
101 if (!lock_hpte(hpte, HPTE_V_HVLOCK | HPTE_V_VALID))
102 return H_PTEG_FULL;
103 }
104 hpte[1] = ptel;
105 eieio();
106 hpte[0] = pteh;
107 asm volatile("ptesync" : : : "memory");
108 atomic_inc(&kvm->arch.ram_pginfo[lpn].refcnt);
109 vcpu->arch.gpr[4] = pte_index + i;
110 return H_SUCCESS;
111}
112
113static unsigned long compute_tlbie_rb(unsigned long v, unsigned long r,
114 unsigned long pte_index)
115{
116 unsigned long rb, va_low;
117
118 rb = (v & ~0x7fUL) << 16; /* AVA field */
119 va_low = pte_index >> 3;
120 if (v & HPTE_V_SECONDARY)
121 va_low = ~va_low;
122 /* xor vsid from AVA */
123 if (!(v & HPTE_V_1TB_SEG))
124 va_low ^= v >> 12;
125 else
126 va_low ^= v >> 24;
127 va_low &= 0x7ff;
128 if (v & HPTE_V_LARGE) {
129 rb |= 1; /* L field */
130 if (cpu_has_feature(CPU_FTR_ARCH_206) &&
131 (r & 0xff000)) {
132 /* non-16MB large page, must be 64k */
133 /* (masks depend on page size) */
134 rb |= 0x1000; /* page encoding in LP field */
135 rb |= (va_low & 0x7f) << 16; /* 7b of VA in AVA/LP field */
136 rb |= (va_low & 0xfe); /* AVAL field (P7 doesn't seem to care) */
137 }
138 } else {
139 /* 4kB page */
140 rb |= (va_low & 0x7ff) << 12; /* remaining 11b of VA */
141 }
142 rb |= (v >> 54) & 0x300; /* B field */
143 return rb;
144}
145
146#define LOCK_TOKEN (*(u32 *)(&get_paca()->lock_token))
147
148static inline int try_lock_tlbie(unsigned int *lock)
149{
150 unsigned int tmp, old;
151 unsigned int token = LOCK_TOKEN;
152
153 asm volatile("1:lwarx %1,0,%2\n"
154 " cmpwi cr0,%1,0\n"
155 " bne 2f\n"
156 " stwcx. %3,0,%2\n"
157 " bne- 1b\n"
158 " isync\n"
159 "2:"
160 : "=&r" (tmp), "=&r" (old)
161 : "r" (lock), "r" (token)
162 : "cc", "memory");
163 return old == 0;
164}
165
166long kvmppc_h_remove(struct kvm_vcpu *vcpu, unsigned long flags,
167 unsigned long pte_index, unsigned long avpn,
168 unsigned long va)
169{
170 struct kvm *kvm = vcpu->kvm;
171 unsigned long *hpte;
172 unsigned long v, r, rb;
173
174 if (pte_index >= (HPT_NPTEG << 3))
175 return H_PARAMETER;
176 hpte = (unsigned long *)(kvm->arch.hpt_virt + (pte_index << 4));
177 while (!lock_hpte(hpte, HPTE_V_HVLOCK))
178 cpu_relax();
179 if ((hpte[0] & HPTE_V_VALID) == 0 ||
180 ((flags & H_AVPN) && (hpte[0] & ~0x7fUL) != avpn) ||
181 ((flags & H_ANDCOND) && (hpte[0] & avpn) != 0)) {
182 hpte[0] &= ~HPTE_V_HVLOCK;
183 return H_NOT_FOUND;
184 }
185 if (atomic_read(&kvm->online_vcpus) == 1)
186 flags |= H_LOCAL;
187 vcpu->arch.gpr[4] = v = hpte[0] & ~HPTE_V_HVLOCK;
188 vcpu->arch.gpr[5] = r = hpte[1];
189 rb = compute_tlbie_rb(v, r, pte_index);
190 hpte[0] = 0;
191 if (!(flags & H_LOCAL)) {
192 while(!try_lock_tlbie(&kvm->arch.tlbie_lock))
193 cpu_relax();
194 asm volatile("ptesync" : : : "memory");
195 asm volatile(PPC_TLBIE(%1,%0)"; eieio; tlbsync"
196 : : "r" (rb), "r" (kvm->arch.lpid));
197 asm volatile("ptesync" : : : "memory");
198 kvm->arch.tlbie_lock = 0;
199 } else {
200 asm volatile("ptesync" : : : "memory");
201 asm volatile("tlbiel %0" : : "r" (rb));
202 asm volatile("ptesync" : : : "memory");
203 }
204 return H_SUCCESS;
205}
206
207long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu)
208{
209 struct kvm *kvm = vcpu->kvm;
210 unsigned long *args = &vcpu->arch.gpr[4];
211 unsigned long *hp, tlbrb[4];
212 long int i, found;
213 long int n_inval = 0;
214 unsigned long flags, req, pte_index;
215 long int local = 0;
216 long int ret = H_SUCCESS;
217
218 if (atomic_read(&kvm->online_vcpus) == 1)
219 local = 1;
220 for (i = 0; i < 4; ++i) {
221 pte_index = args[i * 2];
222 flags = pte_index >> 56;
223 pte_index &= ((1ul << 56) - 1);
224 req = flags >> 6;
225 flags &= 3;
226 if (req == 3)
227 break;
228 if (req != 1 || flags == 3 ||
229 pte_index >= (HPT_NPTEG << 3)) {
230 /* parameter error */
231 args[i * 2] = ((0xa0 | flags) << 56) + pte_index;
232 ret = H_PARAMETER;
233 break;
234 }
235 hp = (unsigned long *)(kvm->arch.hpt_virt + (pte_index << 4));
236 while (!lock_hpte(hp, HPTE_V_HVLOCK))
237 cpu_relax();
238 found = 0;
239 if (hp[0] & HPTE_V_VALID) {
240 switch (flags & 3) {
241 case 0: /* absolute */
242 found = 1;
243 break;
244 case 1: /* andcond */
245 if (!(hp[0] & args[i * 2 + 1]))
246 found = 1;
247 break;
248 case 2: /* AVPN */
249 if ((hp[0] & ~0x7fUL) == args[i * 2 + 1])
250 found = 1;
251 break;
252 }
253 }
254 if (!found) {
255 hp[0] &= ~HPTE_V_HVLOCK;
256 args[i * 2] = ((0x90 | flags) << 56) + pte_index;
257 continue;
258 }
259 /* insert R and C bits from PTE */
260 flags |= (hp[1] >> 5) & 0x0c;
261 args[i * 2] = ((0x80 | flags) << 56) + pte_index;
262 tlbrb[n_inval++] = compute_tlbie_rb(hp[0], hp[1], pte_index);
263 hp[0] = 0;
264 }
265 if (n_inval == 0)
266 return ret;
267
268 if (!local) {
269 while(!try_lock_tlbie(&kvm->arch.tlbie_lock))
270 cpu_relax();
271 asm volatile("ptesync" : : : "memory");
272 for (i = 0; i < n_inval; ++i)
273 asm volatile(PPC_TLBIE(%1,%0)
274 : : "r" (tlbrb[i]), "r" (kvm->arch.lpid));
275 asm volatile("eieio; tlbsync; ptesync" : : : "memory");
276 kvm->arch.tlbie_lock = 0;
277 } else {
278 asm volatile("ptesync" : : : "memory");
279 for (i = 0; i < n_inval; ++i)
280 asm volatile("tlbiel %0" : : "r" (tlbrb[i]));
281 asm volatile("ptesync" : : : "memory");
282 }
283 return ret;
284}
285
286long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags,
287 unsigned long pte_index, unsigned long avpn,
288 unsigned long va)
289{
290 struct kvm *kvm = vcpu->kvm;
291 unsigned long *hpte;
292 unsigned long v, r, rb;
293
294 if (pte_index >= (HPT_NPTEG << 3))
295 return H_PARAMETER;
296 hpte = (unsigned long *)(kvm->arch.hpt_virt + (pte_index << 4));
297 while (!lock_hpte(hpte, HPTE_V_HVLOCK))
298 cpu_relax();
299 if ((hpte[0] & HPTE_V_VALID) == 0 ||
300 ((flags & H_AVPN) && (hpte[0] & ~0x7fUL) != avpn)) {
301 hpte[0] &= ~HPTE_V_HVLOCK;
302 return H_NOT_FOUND;
303 }
304 if (atomic_read(&kvm->online_vcpus) == 1)
305 flags |= H_LOCAL;
306 v = hpte[0];
307 r = hpte[1] & ~(HPTE_R_PP0 | HPTE_R_PP | HPTE_R_N |
308 HPTE_R_KEY_HI | HPTE_R_KEY_LO);
309 r |= (flags << 55) & HPTE_R_PP0;
310 r |= (flags << 48) & HPTE_R_KEY_HI;
311 r |= flags & (HPTE_R_PP | HPTE_R_N | HPTE_R_KEY_LO);
312 rb = compute_tlbie_rb(v, r, pte_index);
313 hpte[0] = v & ~HPTE_V_VALID;
314 if (!(flags & H_LOCAL)) {
315 while(!try_lock_tlbie(&kvm->arch.tlbie_lock))
316 cpu_relax();
317 asm volatile("ptesync" : : : "memory");
318 asm volatile(PPC_TLBIE(%1,%0)"; eieio; tlbsync"
319 : : "r" (rb), "r" (kvm->arch.lpid));
320 asm volatile("ptesync" : : : "memory");
321 kvm->arch.tlbie_lock = 0;
322 } else {
323 asm volatile("ptesync" : : : "memory");
324 asm volatile("tlbiel %0" : : "r" (rb));
325 asm volatile("ptesync" : : : "memory");
326 }
327 hpte[1] = r;
328 eieio();
329 hpte[0] = v & ~HPTE_V_HVLOCK;
330 asm volatile("ptesync" : : : "memory");
331 return H_SUCCESS;
332}
333
334static unsigned long reverse_xlate(struct kvm *kvm, unsigned long realaddr)
335{
336 long int i;
337 unsigned long offset, rpn;
338
339 offset = realaddr & (kvm->arch.ram_psize - 1);
340 rpn = (realaddr - offset) >> PAGE_SHIFT;
341 for (i = 0; i < kvm->arch.ram_npages; ++i)
342 if (rpn == kvm->arch.ram_pginfo[i].pfn)
343 return (i << PAGE_SHIFT) + offset;
344 return HPTE_R_RPN; /* all 1s in the RPN field */
345}
346
347long kvmppc_h_read(struct kvm_vcpu *vcpu, unsigned long flags,
348 unsigned long pte_index)
349{
350 struct kvm *kvm = vcpu->kvm;
351 unsigned long *hpte, r;
352 int i, n = 1;
353
354 if (pte_index >= (HPT_NPTEG << 3))
355 return H_PARAMETER;
356 if (flags & H_READ_4) {
357 pte_index &= ~3;
358 n = 4;
359 }
360 for (i = 0; i < n; ++i, ++pte_index) {
361 hpte = (unsigned long *)(kvm->arch.hpt_virt + (pte_index << 4));
362 r = hpte[1];
363 if ((flags & H_R_XLATE) && (hpte[0] & HPTE_V_VALID))
364 r = reverse_xlate(kvm, r & HPTE_R_RPN) |
365 (r & ~HPTE_R_RPN);
366 vcpu->arch.gpr[4 + i * 2] = hpte[0];
367 vcpu->arch.gpr[5 + i * 2] = r;
368 }
369 return H_SUCCESS;
370}
diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
new file mode 100644
index 000000000000..6dd33581a228
--- /dev/null
+++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
@@ -0,0 +1,1345 @@
1/*
2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License, version 2, as
4 * published by the Free Software Foundation.
5 *
6 * This program is distributed in the hope that it will be useful,
7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9 * GNU General Public License for more details.
10 *
11 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
12 *
13 * Derived from book3s_rmhandlers.S and other files, which are:
14 *
15 * Copyright SUSE Linux Products GmbH 2009
16 *
17 * Authors: Alexander Graf <agraf@suse.de>
18 */
19
20#include <asm/ppc_asm.h>
21#include <asm/kvm_asm.h>
22#include <asm/reg.h>
23#include <asm/page.h>
24#include <asm/asm-offsets.h>
25#include <asm/exception-64s.h>
26
27/*****************************************************************************
28 * *
29 * Real Mode handlers that need to be in the linear mapping *
30 * *
31 ****************************************************************************/
32
33 .globl kvmppc_skip_interrupt
34kvmppc_skip_interrupt:
35 mfspr r13,SPRN_SRR0
36 addi r13,r13,4
37 mtspr SPRN_SRR0,r13
38 GET_SCRATCH0(r13)
39 rfid
40 b .
41
42 .globl kvmppc_skip_Hinterrupt
43kvmppc_skip_Hinterrupt:
44 mfspr r13,SPRN_HSRR0
45 addi r13,r13,4
46 mtspr SPRN_HSRR0,r13
47 GET_SCRATCH0(r13)
48 hrfid
49 b .
50
51/*
52 * Call kvmppc_handler_trampoline_enter in real mode.
53 * Must be called with interrupts hard-disabled.
54 *
55 * Input Registers:
56 *
57 * LR = return address to continue at after eventually re-enabling MMU
58 */
59_GLOBAL(kvmppc_hv_entry_trampoline)
60 mfmsr r10
61 LOAD_REG_ADDR(r5, kvmppc_hv_entry)
62 li r0,MSR_RI
63 andc r0,r10,r0
64 li r6,MSR_IR | MSR_DR
65 andc r6,r10,r6
66 mtmsrd r0,1 /* clear RI in MSR */
67 mtsrr0 r5
68 mtsrr1 r6
69 RFI
70
71#define ULONG_SIZE 8
72#define VCPU_GPR(n) (VCPU_GPRS + (n * ULONG_SIZE))
73
74/******************************************************************************
75 * *
76 * Entry code *
77 * *
78 *****************************************************************************/
79
80#define XICS_XIRR 4
81#define XICS_QIRR 0xc
82
83/*
84 * We come in here when wakened from nap mode on a secondary hw thread.
85 * Relocation is off and most register values are lost.
86 * r13 points to the PACA.
87 */
88 .globl kvm_start_guest
89kvm_start_guest:
90 ld r1,PACAEMERGSP(r13)
91 subi r1,r1,STACK_FRAME_OVERHEAD
92
93 /* get vcpu pointer */
94 ld r4, HSTATE_KVM_VCPU(r13)
95
96 /* We got here with an IPI; clear it */
97 ld r5, HSTATE_XICS_PHYS(r13)
98 li r0, 0xff
99 li r6, XICS_QIRR
100 li r7, XICS_XIRR
101 lwzcix r8, r5, r7 /* ack the interrupt */
102 sync
103 stbcix r0, r5, r6 /* clear it */
104 stwcix r8, r5, r7 /* EOI it */
105
106.global kvmppc_hv_entry
107kvmppc_hv_entry:
108
109 /* Required state:
110 *
111 * R4 = vcpu pointer
112 * MSR = ~IR|DR
113 * R13 = PACA
114 * R1 = host R1
115 * all other volatile GPRS = free
116 */
117 mflr r0
118 std r0, HSTATE_VMHANDLER(r13)
119
120 ld r14, VCPU_GPR(r14)(r4)
121 ld r15, VCPU_GPR(r15)(r4)
122 ld r16, VCPU_GPR(r16)(r4)
123 ld r17, VCPU_GPR(r17)(r4)
124 ld r18, VCPU_GPR(r18)(r4)
125 ld r19, VCPU_GPR(r19)(r4)
126 ld r20, VCPU_GPR(r20)(r4)
127 ld r21, VCPU_GPR(r21)(r4)
128 ld r22, VCPU_GPR(r22)(r4)
129 ld r23, VCPU_GPR(r23)(r4)
130 ld r24, VCPU_GPR(r24)(r4)
131 ld r25, VCPU_GPR(r25)(r4)
132 ld r26, VCPU_GPR(r26)(r4)
133 ld r27, VCPU_GPR(r27)(r4)
134 ld r28, VCPU_GPR(r28)(r4)
135 ld r29, VCPU_GPR(r29)(r4)
136 ld r30, VCPU_GPR(r30)(r4)
137 ld r31, VCPU_GPR(r31)(r4)
138
139 /* Load guest PMU registers */
140 /* R4 is live here (vcpu pointer) */
141 li r3, 1
142 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */
143 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */
144 isync
145 lwz r3, VCPU_PMC(r4) /* always load up guest PMU registers */
146 lwz r5, VCPU_PMC + 4(r4) /* to prevent information leak */
147 lwz r6, VCPU_PMC + 8(r4)
148 lwz r7, VCPU_PMC + 12(r4)
149 lwz r8, VCPU_PMC + 16(r4)
150 lwz r9, VCPU_PMC + 20(r4)
151BEGIN_FTR_SECTION
152 lwz r10, VCPU_PMC + 24(r4)
153 lwz r11, VCPU_PMC + 28(r4)
154END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
155 mtspr SPRN_PMC1, r3
156 mtspr SPRN_PMC2, r5
157 mtspr SPRN_PMC3, r6
158 mtspr SPRN_PMC4, r7
159 mtspr SPRN_PMC5, r8
160 mtspr SPRN_PMC6, r9
161BEGIN_FTR_SECTION
162 mtspr SPRN_PMC7, r10
163 mtspr SPRN_PMC8, r11
164END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
165 ld r3, VCPU_MMCR(r4)
166 ld r5, VCPU_MMCR + 8(r4)
167 ld r6, VCPU_MMCR + 16(r4)
168 mtspr SPRN_MMCR1, r5
169 mtspr SPRN_MMCRA, r6
170 mtspr SPRN_MMCR0, r3
171 isync
172
173 /* Load up FP, VMX and VSX registers */
174 bl kvmppc_load_fp
175
176BEGIN_FTR_SECTION
177 /* Switch DSCR to guest value */
178 ld r5, VCPU_DSCR(r4)
179 mtspr SPRN_DSCR, r5
180END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
181
182 /*
183 * Set the decrementer to the guest decrementer.
184 */
185 ld r8,VCPU_DEC_EXPIRES(r4)
186 mftb r7
187 subf r3,r7,r8
188 mtspr SPRN_DEC,r3
189 stw r3,VCPU_DEC(r4)
190
191 ld r5, VCPU_SPRG0(r4)
192 ld r6, VCPU_SPRG1(r4)
193 ld r7, VCPU_SPRG2(r4)
194 ld r8, VCPU_SPRG3(r4)
195 mtspr SPRN_SPRG0, r5
196 mtspr SPRN_SPRG1, r6
197 mtspr SPRN_SPRG2, r7
198 mtspr SPRN_SPRG3, r8
199
200 /* Save R1 in the PACA */
201 std r1, HSTATE_HOST_R1(r13)
202
203 /* Increment yield count if they have a VPA */
204 ld r3, VCPU_VPA(r4)
205 cmpdi r3, 0
206 beq 25f
207 lwz r5, LPPACA_YIELDCOUNT(r3)
208 addi r5, r5, 1
209 stw r5, LPPACA_YIELDCOUNT(r3)
21025:
211 /* Load up DAR and DSISR */
212 ld r5, VCPU_DAR(r4)
213 lwz r6, VCPU_DSISR(r4)
214 mtspr SPRN_DAR, r5
215 mtspr SPRN_DSISR, r6
216
217 /* Set partition DABR */
218 li r5,3
219 ld r6,VCPU_DABR(r4)
220 mtspr SPRN_DABRX,r5
221 mtspr SPRN_DABR,r6
222
223BEGIN_FTR_SECTION
224 /* Restore AMR and UAMOR, set AMOR to all 1s */
225 ld r5,VCPU_AMR(r4)
226 ld r6,VCPU_UAMOR(r4)
227 li r7,-1
228 mtspr SPRN_AMR,r5
229 mtspr SPRN_UAMOR,r6
230 mtspr SPRN_AMOR,r7
231END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
232
233 /* Clear out SLB */
234 li r6,0
235 slbmte r6,r6
236 slbia
237 ptesync
238
239BEGIN_FTR_SECTION
240 b 30f
241END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
242 /*
243 * POWER7 host -> guest partition switch code.
244 * We don't have to lock against concurrent tlbies,
245 * but we do have to coordinate across hardware threads.
246 */
247 /* Increment entry count iff exit count is zero. */
248 ld r5,HSTATE_KVM_VCORE(r13)
249 addi r9,r5,VCORE_ENTRY_EXIT
25021: lwarx r3,0,r9
251 cmpwi r3,0x100 /* any threads starting to exit? */
252 bge secondary_too_late /* if so we're too late to the party */
253 addi r3,r3,1
254 stwcx. r3,0,r9
255 bne 21b
256
257 /* Primary thread switches to guest partition. */
258 ld r9,VCPU_KVM(r4) /* pointer to struct kvm */
259 lwz r6,VCPU_PTID(r4)
260 cmpwi r6,0
261 bne 20f
262 ld r6,KVM_SDR1(r9)
263 lwz r7,KVM_LPID(r9)
264 li r0,LPID_RSVD /* switch to reserved LPID */
265 mtspr SPRN_LPID,r0
266 ptesync
267 mtspr SPRN_SDR1,r6 /* switch to partition page table */
268 mtspr SPRN_LPID,r7
269 isync
270 li r0,1
271 stb r0,VCORE_IN_GUEST(r5) /* signal secondaries to continue */
272 b 10f
273
274 /* Secondary threads wait for primary to have done partition switch */
27520: lbz r0,VCORE_IN_GUEST(r5)
276 cmpwi r0,0
277 beq 20b
278
279 /* Set LPCR. Set the MER bit if there is a pending external irq. */
28010: ld r8,KVM_LPCR(r9)
281 ld r0,VCPU_PENDING_EXC(r4)
282 li r7,(1 << BOOK3S_IRQPRIO_EXTERNAL)
283 oris r7,r7,(1 << BOOK3S_IRQPRIO_EXTERNAL_LEVEL)@h
284 and. r0,r0,r7
285 beq 11f
286 ori r8,r8,LPCR_MER
28711: mtspr SPRN_LPCR,r8
288 ld r8,KVM_RMOR(r9)
289 mtspr SPRN_RMOR,r8
290 isync
291
292 /* Check if HDEC expires soon */
293 mfspr r3,SPRN_HDEC
294 cmpwi r3,10
295 li r12,BOOK3S_INTERRUPT_HV_DECREMENTER
296 mr r9,r4
297 blt hdec_soon
298
299 /*
300 * Invalidate the TLB if we could possibly have stale TLB
301 * entries for this partition on this core due to the use
302 * of tlbiel.
303 * XXX maybe only need this on primary thread?
304 */
305 ld r9,VCPU_KVM(r4) /* pointer to struct kvm */
306 lwz r5,VCPU_VCPUID(r4)
307 lhz r6,PACAPACAINDEX(r13)
308 rldimi r6,r5,0,62 /* XXX map as if threads 1:1 p:v */
309 lhz r8,VCPU_LAST_CPU(r4)
310 sldi r7,r6,1 /* see if this is the same vcpu */
311 add r7,r7,r9 /* as last ran on this pcpu */
312 lhz r0,KVM_LAST_VCPU(r7)
313 cmpw r6,r8 /* on the same cpu core as last time? */
314 bne 3f
315 cmpw r0,r5 /* same vcpu as this core last ran? */
316 beq 1f
3173: sth r6,VCPU_LAST_CPU(r4) /* if not, invalidate partition TLB */
318 sth r5,KVM_LAST_VCPU(r7)
319 li r6,128
320 mtctr r6
321 li r7,0x800 /* IS field = 0b10 */
322 ptesync
3232: tlbiel r7
324 addi r7,r7,0x1000
325 bdnz 2b
326 ptesync
3271:
328
329 /* Save purr/spurr */
330 mfspr r5,SPRN_PURR
331 mfspr r6,SPRN_SPURR
332 std r5,HSTATE_PURR(r13)
333 std r6,HSTATE_SPURR(r13)
334 ld r7,VCPU_PURR(r4)
335 ld r8,VCPU_SPURR(r4)
336 mtspr SPRN_PURR,r7
337 mtspr SPRN_SPURR,r8
338 b 31f
339
340 /*
341 * PPC970 host -> guest partition switch code.
342 * We have to lock against concurrent tlbies,
343 * using native_tlbie_lock to lock against host tlbies
344 * and kvm->arch.tlbie_lock to lock against guest tlbies.
345 * We also have to invalidate the TLB since its
346 * entries aren't tagged with the LPID.
347 */
34830: ld r9,VCPU_KVM(r4) /* pointer to struct kvm */
349
350 /* first take native_tlbie_lock */
351 .section ".toc","aw"
352toc_tlbie_lock:
353 .tc native_tlbie_lock[TC],native_tlbie_lock
354 .previous
355 ld r3,toc_tlbie_lock@toc(2)
356 lwz r8,PACA_LOCK_TOKEN(r13)
35724: lwarx r0,0,r3
358 cmpwi r0,0
359 bne 24b
360 stwcx. r8,0,r3
361 bne 24b
362 isync
363
364 ld r7,KVM_LPCR(r9) /* use kvm->arch.lpcr to store HID4 */
365 li r0,0x18f
366 rotldi r0,r0,HID4_LPID5_SH /* all lpid bits in HID4 = 1 */
367 or r0,r7,r0
368 ptesync
369 sync
370 mtspr SPRN_HID4,r0 /* switch to reserved LPID */
371 isync
372 li r0,0
373 stw r0,0(r3) /* drop native_tlbie_lock */
374
375 /* invalidate the whole TLB */
376 li r0,256
377 mtctr r0
378 li r6,0
37925: tlbiel r6
380 addi r6,r6,0x1000
381 bdnz 25b
382 ptesync
383
384 /* Take the guest's tlbie_lock */
385 addi r3,r9,KVM_TLBIE_LOCK
38624: lwarx r0,0,r3
387 cmpwi r0,0
388 bne 24b
389 stwcx. r8,0,r3
390 bne 24b
391 isync
392 ld r6,KVM_SDR1(r9)
393 mtspr SPRN_SDR1,r6 /* switch to partition page table */
394
395 /* Set up HID4 with the guest's LPID etc. */
396 sync
397 mtspr SPRN_HID4,r7
398 isync
399
400 /* drop the guest's tlbie_lock */
401 li r0,0
402 stw r0,0(r3)
403
404 /* Check if HDEC expires soon */
405 mfspr r3,SPRN_HDEC
406 cmpwi r3,10
407 li r12,BOOK3S_INTERRUPT_HV_DECREMENTER
408 mr r9,r4
409 blt hdec_soon
410
411 /* Enable HDEC interrupts */
412 mfspr r0,SPRN_HID0
413 li r3,1
414 rldimi r0,r3, HID0_HDICE_SH, 64-HID0_HDICE_SH-1
415 sync
416 mtspr SPRN_HID0,r0
417 mfspr r0,SPRN_HID0
418 mfspr r0,SPRN_HID0
419 mfspr r0,SPRN_HID0
420 mfspr r0,SPRN_HID0
421 mfspr r0,SPRN_HID0
422 mfspr r0,SPRN_HID0
423
424 /* Load up guest SLB entries */
42531: lwz r5,VCPU_SLB_MAX(r4)
426 cmpwi r5,0
427 beq 9f
428 mtctr r5
429 addi r6,r4,VCPU_SLB
4301: ld r8,VCPU_SLB_E(r6)
431 ld r9,VCPU_SLB_V(r6)
432 slbmte r9,r8
433 addi r6,r6,VCPU_SLB_SIZE
434 bdnz 1b
4359:
436
437 /* Restore state of CTRL run bit; assume 1 on entry */
438 lwz r5,VCPU_CTRL(r4)
439 andi. r5,r5,1
440 bne 4f
441 mfspr r6,SPRN_CTRLF
442 clrrdi r6,r6,1
443 mtspr SPRN_CTRLT,r6
4444:
445 ld r6, VCPU_CTR(r4)
446 lwz r7, VCPU_XER(r4)
447
448 mtctr r6
449 mtxer r7
450
451 /* Move SRR0 and SRR1 into the respective regs */
452 ld r6, VCPU_SRR0(r4)
453 ld r7, VCPU_SRR1(r4)
454 mtspr SPRN_SRR0, r6
455 mtspr SPRN_SRR1, r7
456
457 ld r10, VCPU_PC(r4)
458
459 ld r11, VCPU_MSR(r4) /* r10 = vcpu->arch.msr & ~MSR_HV */
460 rldicl r11, r11, 63 - MSR_HV_LG, 1
461 rotldi r11, r11, 1 + MSR_HV_LG
462 ori r11, r11, MSR_ME
463
464fast_guest_return:
465 mtspr SPRN_HSRR0,r10
466 mtspr SPRN_HSRR1,r11
467
468 /* Activate guest mode, so faults get handled by KVM */
469 li r9, KVM_GUEST_MODE_GUEST
470 stb r9, HSTATE_IN_GUEST(r13)
471
472 /* Enter guest */
473
474 ld r5, VCPU_LR(r4)
475 lwz r6, VCPU_CR(r4)
476 mtlr r5
477 mtcr r6
478
479 ld r0, VCPU_GPR(r0)(r4)
480 ld r1, VCPU_GPR(r1)(r4)
481 ld r2, VCPU_GPR(r2)(r4)
482 ld r3, VCPU_GPR(r3)(r4)
483 ld r5, VCPU_GPR(r5)(r4)
484 ld r6, VCPU_GPR(r6)(r4)
485 ld r7, VCPU_GPR(r7)(r4)
486 ld r8, VCPU_GPR(r8)(r4)
487 ld r9, VCPU_GPR(r9)(r4)
488 ld r10, VCPU_GPR(r10)(r4)
489 ld r11, VCPU_GPR(r11)(r4)
490 ld r12, VCPU_GPR(r12)(r4)
491 ld r13, VCPU_GPR(r13)(r4)
492
493 ld r4, VCPU_GPR(r4)(r4)
494
495 hrfid
496 b .
497
498/******************************************************************************
499 * *
500 * Exit code *
501 * *
502 *****************************************************************************/
503
504/*
505 * We come here from the first-level interrupt handlers.
506 */
507 .globl kvmppc_interrupt
508kvmppc_interrupt:
509 /*
510 * Register contents:
511 * R12 = interrupt vector
512 * R13 = PACA
513 * guest CR, R12 saved in shadow VCPU SCRATCH1/0
514 * guest R13 saved in SPRN_SCRATCH0
515 */
516 /* abuse host_r2 as third scratch area; we get r2 from PACATOC(r13) */
517 std r9, HSTATE_HOST_R2(r13)
518 ld r9, HSTATE_KVM_VCPU(r13)
519
520 /* Save registers */
521
522 std r0, VCPU_GPR(r0)(r9)
523 std r1, VCPU_GPR(r1)(r9)
524 std r2, VCPU_GPR(r2)(r9)
525 std r3, VCPU_GPR(r3)(r9)
526 std r4, VCPU_GPR(r4)(r9)
527 std r5, VCPU_GPR(r5)(r9)
528 std r6, VCPU_GPR(r6)(r9)
529 std r7, VCPU_GPR(r7)(r9)
530 std r8, VCPU_GPR(r8)(r9)
531 ld r0, HSTATE_HOST_R2(r13)
532 std r0, VCPU_GPR(r9)(r9)
533 std r10, VCPU_GPR(r10)(r9)
534 std r11, VCPU_GPR(r11)(r9)
535 ld r3, HSTATE_SCRATCH0(r13)
536 lwz r4, HSTATE_SCRATCH1(r13)
537 std r3, VCPU_GPR(r12)(r9)
538 stw r4, VCPU_CR(r9)
539
540 /* Restore R1/R2 so we can handle faults */
541 ld r1, HSTATE_HOST_R1(r13)
542 ld r2, PACATOC(r13)
543
544 mfspr r10, SPRN_SRR0
545 mfspr r11, SPRN_SRR1
546 std r10, VCPU_SRR0(r9)
547 std r11, VCPU_SRR1(r9)
548 andi. r0, r12, 2 /* need to read HSRR0/1? */
549 beq 1f
550 mfspr r10, SPRN_HSRR0
551 mfspr r11, SPRN_HSRR1
552 clrrdi r12, r12, 2
5531: std r10, VCPU_PC(r9)
554 std r11, VCPU_MSR(r9)
555
556 GET_SCRATCH0(r3)
557 mflr r4
558 std r3, VCPU_GPR(r13)(r9)
559 std r4, VCPU_LR(r9)
560
561 /* Unset guest mode */
562 li r0, KVM_GUEST_MODE_NONE
563 stb r0, HSTATE_IN_GUEST(r13)
564
565 stw r12,VCPU_TRAP(r9)
566
567 /* See if this is a leftover HDEC interrupt */
568 cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER
569 bne 2f
570 mfspr r3,SPRN_HDEC
571 cmpwi r3,0
572 bge ignore_hdec
5732:
574 /* See if this is something we can handle in real mode */
575 cmpwi r12,BOOK3S_INTERRUPT_SYSCALL
576 beq hcall_try_real_mode
577hcall_real_cont:
578
579 /* Check for mediated interrupts (could be done earlier really ...) */
580BEGIN_FTR_SECTION
581 cmpwi r12,BOOK3S_INTERRUPT_EXTERNAL
582 bne+ 1f
583 ld r5,VCPU_KVM(r9)
584 ld r5,KVM_LPCR(r5)
585 andi. r0,r11,MSR_EE
586 beq 1f
587 andi. r0,r5,LPCR_MER
588 bne bounce_ext_interrupt
5891:
590END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
591
592 /* Save DEC */
593 mfspr r5,SPRN_DEC
594 mftb r6
595 extsw r5,r5
596 add r5,r5,r6
597 std r5,VCPU_DEC_EXPIRES(r9)
598
599 /* Save HEIR (HV emulation assist reg) in last_inst
600 if this is an HEI (HV emulation interrupt, e40) */
601 li r3,-1
602BEGIN_FTR_SECTION
603 cmpwi r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST
604 bne 11f
605 mfspr r3,SPRN_HEIR
606END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
60711: stw r3,VCPU_LAST_INST(r9)
608
609 /* Save more register state */
610 mfxer r5
611 mfdar r6
612 mfdsisr r7
613 mfctr r8
614
615 stw r5, VCPU_XER(r9)
616 std r6, VCPU_DAR(r9)
617 stw r7, VCPU_DSISR(r9)
618 std r8, VCPU_CTR(r9)
619 /* grab HDAR & HDSISR if HV data storage interrupt (HDSI) */
620BEGIN_FTR_SECTION
621 cmpwi r12,BOOK3S_INTERRUPT_H_DATA_STORAGE
622 beq 6f
623END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
6247: std r6, VCPU_FAULT_DAR(r9)
625 stw r7, VCPU_FAULT_DSISR(r9)
626
627 /* Save guest CTRL register, set runlatch to 1 */
628 mfspr r6,SPRN_CTRLF
629 stw r6,VCPU_CTRL(r9)
630 andi. r0,r6,1
631 bne 4f
632 ori r6,r6,1
633 mtspr SPRN_CTRLT,r6
6344:
635 /* Read the guest SLB and save it away */
636 lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */
637 mtctr r0
638 li r6,0
639 addi r7,r9,VCPU_SLB
640 li r5,0
6411: slbmfee r8,r6
642 andis. r0,r8,SLB_ESID_V@h
643 beq 2f
644 add r8,r8,r6 /* put index in */
645 slbmfev r3,r6
646 std r8,VCPU_SLB_E(r7)
647 std r3,VCPU_SLB_V(r7)
648 addi r7,r7,VCPU_SLB_SIZE
649 addi r5,r5,1
6502: addi r6,r6,1
651 bdnz 1b
652 stw r5,VCPU_SLB_MAX(r9)
653
654 /*
655 * Save the guest PURR/SPURR
656 */
657BEGIN_FTR_SECTION
658 mfspr r5,SPRN_PURR
659 mfspr r6,SPRN_SPURR
660 ld r7,VCPU_PURR(r9)
661 ld r8,VCPU_SPURR(r9)
662 std r5,VCPU_PURR(r9)
663 std r6,VCPU_SPURR(r9)
664 subf r5,r7,r5
665 subf r6,r8,r6
666
667 /*
668 * Restore host PURR/SPURR and add guest times
669 * so that the time in the guest gets accounted.
670 */
671 ld r3,HSTATE_PURR(r13)
672 ld r4,HSTATE_SPURR(r13)
673 add r3,r3,r5
674 add r4,r4,r6
675 mtspr SPRN_PURR,r3
676 mtspr SPRN_SPURR,r4
677END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_201)
678
679 /* Clear out SLB */
680 li r5,0
681 slbmte r5,r5
682 slbia
683 ptesync
684
685hdec_soon:
686BEGIN_FTR_SECTION
687 b 32f
688END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
689 /*
690 * POWER7 guest -> host partition switch code.
691 * We don't have to lock against tlbies but we do
692 * have to coordinate the hardware threads.
693 */
694 /* Increment the threads-exiting-guest count in the 0xff00
695 bits of vcore->entry_exit_count */
696 lwsync
697 ld r5,HSTATE_KVM_VCORE(r13)
698 addi r6,r5,VCORE_ENTRY_EXIT
69941: lwarx r3,0,r6
700 addi r0,r3,0x100
701 stwcx. r0,0,r6
702 bne 41b
703
704 /*
705 * At this point we have an interrupt that we have to pass
706 * up to the kernel or qemu; we can't handle it in real mode.
707 * Thus we have to do a partition switch, so we have to
708 * collect the other threads, if we are the first thread
709 * to take an interrupt. To do this, we set the HDEC to 0,
710 * which causes an HDEC interrupt in all threads within 2ns
711 * because the HDEC register is shared between all 4 threads.
712 * However, we don't need to bother if this is an HDEC
713 * interrupt, since the other threads will already be on their
714 * way here in that case.
715 */
716 cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER
717 beq 40f
718 cmpwi r3,0x100 /* Are we the first here? */
719 bge 40f
720 cmpwi r3,1
721 ble 40f
722 li r0,0
723 mtspr SPRN_HDEC,r0
72440:
725
726 /* Secondary threads wait for primary to do partition switch */
727 ld r4,VCPU_KVM(r9) /* pointer to struct kvm */
728 ld r5,HSTATE_KVM_VCORE(r13)
729 lwz r3,VCPU_PTID(r9)
730 cmpwi r3,0
731 beq 15f
732 HMT_LOW
73313: lbz r3,VCORE_IN_GUEST(r5)
734 cmpwi r3,0
735 bne 13b
736 HMT_MEDIUM
737 b 16f
738
739 /* Primary thread waits for all the secondaries to exit guest */
74015: lwz r3,VCORE_ENTRY_EXIT(r5)
741 srwi r0,r3,8
742 clrldi r3,r3,56
743 cmpw r3,r0
744 bne 15b
745 isync
746
747 /* Primary thread switches back to host partition */
748 ld r6,KVM_HOST_SDR1(r4)
749 lwz r7,KVM_HOST_LPID(r4)
750 li r8,LPID_RSVD /* switch to reserved LPID */
751 mtspr SPRN_LPID,r8
752 ptesync
753 mtspr SPRN_SDR1,r6 /* switch to partition page table */
754 mtspr SPRN_LPID,r7
755 isync
756 li r0,0
757 stb r0,VCORE_IN_GUEST(r5)
758 lis r8,0x7fff /* MAX_INT@h */
759 mtspr SPRN_HDEC,r8
760
76116: ld r8,KVM_HOST_LPCR(r4)
762 mtspr SPRN_LPCR,r8
763 isync
764 b 33f
765
766 /*
767 * PPC970 guest -> host partition switch code.
768 * We have to lock against concurrent tlbies, and
769 * we have to flush the whole TLB.
770 */
77132: ld r4,VCPU_KVM(r9) /* pointer to struct kvm */
772
773 /* Take the guest's tlbie_lock */
774 lwz r8,PACA_LOCK_TOKEN(r13)
775 addi r3,r4,KVM_TLBIE_LOCK
77624: lwarx r0,0,r3
777 cmpwi r0,0
778 bne 24b
779 stwcx. r8,0,r3
780 bne 24b
781 isync
782
783 ld r7,KVM_HOST_LPCR(r4) /* use kvm->arch.host_lpcr for HID4 */
784 li r0,0x18f
785 rotldi r0,r0,HID4_LPID5_SH /* all lpid bits in HID4 = 1 */
786 or r0,r7,r0
787 ptesync
788 sync
789 mtspr SPRN_HID4,r0 /* switch to reserved LPID */
790 isync
791 li r0,0
792 stw r0,0(r3) /* drop guest tlbie_lock */
793
794 /* invalidate the whole TLB */
795 li r0,256
796 mtctr r0
797 li r6,0
79825: tlbiel r6
799 addi r6,r6,0x1000
800 bdnz 25b
801 ptesync
802
803 /* take native_tlbie_lock */
804 ld r3,toc_tlbie_lock@toc(2)
80524: lwarx r0,0,r3
806 cmpwi r0,0
807 bne 24b
808 stwcx. r8,0,r3
809 bne 24b
810 isync
811
812 ld r6,KVM_HOST_SDR1(r4)
813 mtspr SPRN_SDR1,r6 /* switch to host page table */
814
815 /* Set up host HID4 value */
816 sync
817 mtspr SPRN_HID4,r7
818 isync
819 li r0,0
820 stw r0,0(r3) /* drop native_tlbie_lock */
821
822 lis r8,0x7fff /* MAX_INT@h */
823 mtspr SPRN_HDEC,r8
824
825 /* Disable HDEC interrupts */
826 mfspr r0,SPRN_HID0
827 li r3,0
828 rldimi r0,r3, HID0_HDICE_SH, 64-HID0_HDICE_SH-1
829 sync
830 mtspr SPRN_HID0,r0
831 mfspr r0,SPRN_HID0
832 mfspr r0,SPRN_HID0
833 mfspr r0,SPRN_HID0
834 mfspr r0,SPRN_HID0
835 mfspr r0,SPRN_HID0
836 mfspr r0,SPRN_HID0
837
838 /* load host SLB entries */
83933: ld r8,PACA_SLBSHADOWPTR(r13)
840
841 .rept SLB_NUM_BOLTED
842 ld r5,SLBSHADOW_SAVEAREA(r8)
843 ld r6,SLBSHADOW_SAVEAREA+8(r8)
844 andis. r7,r5,SLB_ESID_V@h
845 beq 1f
846 slbmte r6,r5
8471: addi r8,r8,16
848 .endr
849
850 /* Save and reset AMR and UAMOR before turning on the MMU */
851BEGIN_FTR_SECTION
852 mfspr r5,SPRN_AMR
853 mfspr r6,SPRN_UAMOR
854 std r5,VCPU_AMR(r9)
855 std r6,VCPU_UAMOR(r9)
856 li r6,0
857 mtspr SPRN_AMR,r6
858END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
859
860 /* Restore host DABR and DABRX */
861 ld r5,HSTATE_DABR(r13)
862 li r6,7
863 mtspr SPRN_DABR,r5
864 mtspr SPRN_DABRX,r6
865
866 /* Switch DSCR back to host value */
867BEGIN_FTR_SECTION
868 mfspr r8, SPRN_DSCR
869 ld r7, HSTATE_DSCR(r13)
870 std r8, VCPU_DSCR(r7)
871 mtspr SPRN_DSCR, r7
872END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
873
874 /* Save non-volatile GPRs */
875 std r14, VCPU_GPR(r14)(r9)
876 std r15, VCPU_GPR(r15)(r9)
877 std r16, VCPU_GPR(r16)(r9)
878 std r17, VCPU_GPR(r17)(r9)
879 std r18, VCPU_GPR(r18)(r9)
880 std r19, VCPU_GPR(r19)(r9)
881 std r20, VCPU_GPR(r20)(r9)
882 std r21, VCPU_GPR(r21)(r9)
883 std r22, VCPU_GPR(r22)(r9)
884 std r23, VCPU_GPR(r23)(r9)
885 std r24, VCPU_GPR(r24)(r9)
886 std r25, VCPU_GPR(r25)(r9)
887 std r26, VCPU_GPR(r26)(r9)
888 std r27, VCPU_GPR(r27)(r9)
889 std r28, VCPU_GPR(r28)(r9)
890 std r29, VCPU_GPR(r29)(r9)
891 std r30, VCPU_GPR(r30)(r9)
892 std r31, VCPU_GPR(r31)(r9)
893
894 /* Save SPRGs */
895 mfspr r3, SPRN_SPRG0
896 mfspr r4, SPRN_SPRG1
897 mfspr r5, SPRN_SPRG2
898 mfspr r6, SPRN_SPRG3
899 std r3, VCPU_SPRG0(r9)
900 std r4, VCPU_SPRG1(r9)
901 std r5, VCPU_SPRG2(r9)
902 std r6, VCPU_SPRG3(r9)
903
904 /* Increment yield count if they have a VPA */
905 ld r8, VCPU_VPA(r9) /* do they have a VPA? */
906 cmpdi r8, 0
907 beq 25f
908 lwz r3, LPPACA_YIELDCOUNT(r8)
909 addi r3, r3, 1
910 stw r3, LPPACA_YIELDCOUNT(r8)
91125:
912 /* Save PMU registers if requested */
913 /* r8 and cr0.eq are live here */
914 li r3, 1
915 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */
916 mfspr r4, SPRN_MMCR0 /* save MMCR0 */
917 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */
918 isync
919 beq 21f /* if no VPA, save PMU stuff anyway */
920 lbz r7, LPPACA_PMCINUSE(r8)
921 cmpwi r7, 0 /* did they ask for PMU stuff to be saved? */
922 bne 21f
923 std r3, VCPU_MMCR(r9) /* if not, set saved MMCR0 to FC */
924 b 22f
92521: mfspr r5, SPRN_MMCR1
926 mfspr r6, SPRN_MMCRA
927 std r4, VCPU_MMCR(r9)
928 std r5, VCPU_MMCR + 8(r9)
929 std r6, VCPU_MMCR + 16(r9)
930 mfspr r3, SPRN_PMC1
931 mfspr r4, SPRN_PMC2
932 mfspr r5, SPRN_PMC3
933 mfspr r6, SPRN_PMC4
934 mfspr r7, SPRN_PMC5
935 mfspr r8, SPRN_PMC6
936BEGIN_FTR_SECTION
937 mfspr r10, SPRN_PMC7
938 mfspr r11, SPRN_PMC8
939END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
940 stw r3, VCPU_PMC(r9)
941 stw r4, VCPU_PMC + 4(r9)
942 stw r5, VCPU_PMC + 8(r9)
943 stw r6, VCPU_PMC + 12(r9)
944 stw r7, VCPU_PMC + 16(r9)
945 stw r8, VCPU_PMC + 20(r9)
946BEGIN_FTR_SECTION
947 stw r10, VCPU_PMC + 24(r9)
948 stw r11, VCPU_PMC + 28(r9)
949END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
95022:
951 /* save FP state */
952 mr r3, r9
953 bl .kvmppc_save_fp
954
955 /* Secondary threads go off to take a nap on POWER7 */
956BEGIN_FTR_SECTION
957 lwz r0,VCPU_PTID(r3)
958 cmpwi r0,0
959 bne secondary_nap
960END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
961
962 /*
963 * Reload DEC. HDEC interrupts were disabled when
964 * we reloaded the host's LPCR value.
965 */
966 ld r3, HSTATE_DECEXP(r13)
967 mftb r4
968 subf r4, r4, r3
969 mtspr SPRN_DEC, r4
970
971 /* Reload the host's PMU registers */
972 ld r3, PACALPPACAPTR(r13) /* is the host using the PMU? */
973 lbz r4, LPPACA_PMCINUSE(r3)
974 cmpwi r4, 0
975 beq 23f /* skip if not */
976 lwz r3, HSTATE_PMC(r13)
977 lwz r4, HSTATE_PMC + 4(r13)
978 lwz r5, HSTATE_PMC + 8(r13)
979 lwz r6, HSTATE_PMC + 12(r13)
980 lwz r8, HSTATE_PMC + 16(r13)
981 lwz r9, HSTATE_PMC + 20(r13)
982BEGIN_FTR_SECTION
983 lwz r10, HSTATE_PMC + 24(r13)
984 lwz r11, HSTATE_PMC + 28(r13)
985END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
986 mtspr SPRN_PMC1, r3
987 mtspr SPRN_PMC2, r4
988 mtspr SPRN_PMC3, r5
989 mtspr SPRN_PMC4, r6
990 mtspr SPRN_PMC5, r8
991 mtspr SPRN_PMC6, r9
992BEGIN_FTR_SECTION
993 mtspr SPRN_PMC7, r10
994 mtspr SPRN_PMC8, r11
995END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
996 ld r3, HSTATE_MMCR(r13)
997 ld r4, HSTATE_MMCR + 8(r13)
998 ld r5, HSTATE_MMCR + 16(r13)
999 mtspr SPRN_MMCR1, r4
1000 mtspr SPRN_MMCRA, r5
1001 mtspr SPRN_MMCR0, r3
1002 isync
100323:
1004 /*
1005 * For external and machine check interrupts, we need
1006 * to call the Linux handler to process the interrupt.
1007 * We do that by jumping to the interrupt vector address
1008 * which we have in r12. The [h]rfid at the end of the
1009 * handler will return to the book3s_hv_interrupts.S code.
1010 * For other interrupts we do the rfid to get back
1011 * to the book3s_interrupts.S code here.
1012 */
1013 ld r8, HSTATE_VMHANDLER(r13)
1014 ld r7, HSTATE_HOST_MSR(r13)
1015
1016 cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL
1017 beq 11f
1018 cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK
1019
1020 /* RFI into the highmem handler, or branch to interrupt handler */
102112: mfmsr r6
1022 mtctr r12
1023 li r0, MSR_RI
1024 andc r6, r6, r0
1025 mtmsrd r6, 1 /* Clear RI in MSR */
1026 mtsrr0 r8
1027 mtsrr1 r7
1028 beqctr
1029 RFI
1030
103111:
1032BEGIN_FTR_SECTION
1033 b 12b
1034END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
1035 mtspr SPRN_HSRR0, r8
1036 mtspr SPRN_HSRR1, r7
1037 ba 0x500
1038
10396: mfspr r6,SPRN_HDAR
1040 mfspr r7,SPRN_HDSISR
1041 b 7b
1042
1043/*
1044 * Try to handle an hcall in real mode.
1045 * Returns to the guest if we handle it, or continues on up to
1046 * the kernel if we can't (i.e. if we don't have a handler for
1047 * it, or if the handler returns H_TOO_HARD).
1048 */
1049 .globl hcall_try_real_mode
1050hcall_try_real_mode:
1051 ld r3,VCPU_GPR(r3)(r9)
1052 andi. r0,r11,MSR_PR
1053 bne hcall_real_cont
1054 clrrdi r3,r3,2
1055 cmpldi r3,hcall_real_table_end - hcall_real_table
1056 bge hcall_real_cont
1057 LOAD_REG_ADDR(r4, hcall_real_table)
1058 lwzx r3,r3,r4
1059 cmpwi r3,0
1060 beq hcall_real_cont
1061 add r3,r3,r4
1062 mtctr r3
1063 mr r3,r9 /* get vcpu pointer */
1064 ld r4,VCPU_GPR(r4)(r9)
1065 bctrl
1066 cmpdi r3,H_TOO_HARD
1067 beq hcall_real_fallback
1068 ld r4,HSTATE_KVM_VCPU(r13)
1069 std r3,VCPU_GPR(r3)(r4)
1070 ld r10,VCPU_PC(r4)
1071 ld r11,VCPU_MSR(r4)
1072 b fast_guest_return
1073
1074 /* We've attempted a real mode hcall, but it's punted it back
1075 * to userspace. We need to restore some clobbered volatiles
1076 * before resuming the pass-it-to-qemu path */
1077hcall_real_fallback:
1078 li r12,BOOK3S_INTERRUPT_SYSCALL
1079 ld r9, HSTATE_KVM_VCPU(r13)
1080 ld r11, VCPU_MSR(r9)
1081
1082 b hcall_real_cont
1083
1084 .globl hcall_real_table
1085hcall_real_table:
1086 .long 0 /* 0 - unused */
1087 .long .kvmppc_h_remove - hcall_real_table
1088 .long .kvmppc_h_enter - hcall_real_table
1089 .long .kvmppc_h_read - hcall_real_table
1090 .long 0 /* 0x10 - H_CLEAR_MOD */
1091 .long 0 /* 0x14 - H_CLEAR_REF */
1092 .long .kvmppc_h_protect - hcall_real_table
1093 .long 0 /* 0x1c - H_GET_TCE */
1094 .long .kvmppc_h_put_tce - hcall_real_table
1095 .long 0 /* 0x24 - H_SET_SPRG0 */
1096 .long .kvmppc_h_set_dabr - hcall_real_table
1097 .long 0 /* 0x2c */
1098 .long 0 /* 0x30 */
1099 .long 0 /* 0x34 */
1100 .long 0 /* 0x38 */
1101 .long 0 /* 0x3c */
1102 .long 0 /* 0x40 */
1103 .long 0 /* 0x44 */
1104 .long 0 /* 0x48 */
1105 .long 0 /* 0x4c */
1106 .long 0 /* 0x50 */
1107 .long 0 /* 0x54 */
1108 .long 0 /* 0x58 */
1109 .long 0 /* 0x5c */
1110 .long 0 /* 0x60 */
1111 .long 0 /* 0x64 */
1112 .long 0 /* 0x68 */
1113 .long 0 /* 0x6c */
1114 .long 0 /* 0x70 */
1115 .long 0 /* 0x74 */
1116 .long 0 /* 0x78 */
1117 .long 0 /* 0x7c */
1118 .long 0 /* 0x80 */
1119 .long 0 /* 0x84 */
1120 .long 0 /* 0x88 */
1121 .long 0 /* 0x8c */
1122 .long 0 /* 0x90 */
1123 .long 0 /* 0x94 */
1124 .long 0 /* 0x98 */
1125 .long 0 /* 0x9c */
1126 .long 0 /* 0xa0 */
1127 .long 0 /* 0xa4 */
1128 .long 0 /* 0xa8 */
1129 .long 0 /* 0xac */
1130 .long 0 /* 0xb0 */
1131 .long 0 /* 0xb4 */
1132 .long 0 /* 0xb8 */
1133 .long 0 /* 0xbc */
1134 .long 0 /* 0xc0 */
1135 .long 0 /* 0xc4 */
1136 .long 0 /* 0xc8 */
1137 .long 0 /* 0xcc */
1138 .long 0 /* 0xd0 */
1139 .long 0 /* 0xd4 */
1140 .long 0 /* 0xd8 */
1141 .long 0 /* 0xdc */
1142 .long 0 /* 0xe0 */
1143 .long 0 /* 0xe4 */
1144 .long 0 /* 0xe8 */
1145 .long 0 /* 0xec */
1146 .long 0 /* 0xf0 */
1147 .long 0 /* 0xf4 */
1148 .long 0 /* 0xf8 */
1149 .long 0 /* 0xfc */
1150 .long 0 /* 0x100 */
1151 .long 0 /* 0x104 */
1152 .long 0 /* 0x108 */
1153 .long 0 /* 0x10c */
1154 .long 0 /* 0x110 */
1155 .long 0 /* 0x114 */
1156 .long 0 /* 0x118 */
1157 .long 0 /* 0x11c */
1158 .long 0 /* 0x120 */
1159 .long .kvmppc_h_bulk_remove - hcall_real_table
1160hcall_real_table_end:
1161
1162ignore_hdec:
1163 mr r4,r9
1164 b fast_guest_return
1165
1166bounce_ext_interrupt:
1167 mr r4,r9
1168 mtspr SPRN_SRR0,r10
1169 mtspr SPRN_SRR1,r11
1170 li r10,BOOK3S_INTERRUPT_EXTERNAL
1171 LOAD_REG_IMMEDIATE(r11,MSR_SF | MSR_ME);
1172 b fast_guest_return
1173
1174_GLOBAL(kvmppc_h_set_dabr)
1175 std r4,VCPU_DABR(r3)
1176 mtspr SPRN_DABR,r4
1177 li r3,0
1178 blr
1179
1180secondary_too_late:
1181 ld r5,HSTATE_KVM_VCORE(r13)
1182 HMT_LOW
118313: lbz r3,VCORE_IN_GUEST(r5)
1184 cmpwi r3,0
1185 bne 13b
1186 HMT_MEDIUM
1187 ld r11,PACA_SLBSHADOWPTR(r13)
1188
1189 .rept SLB_NUM_BOLTED
1190 ld r5,SLBSHADOW_SAVEAREA(r11)
1191 ld r6,SLBSHADOW_SAVEAREA+8(r11)
1192 andis. r7,r5,SLB_ESID_V@h
1193 beq 1f
1194 slbmte r6,r5
11951: addi r11,r11,16
1196 .endr
1197 b 50f
1198
1199secondary_nap:
1200 /* Clear any pending IPI */
120150: ld r5, HSTATE_XICS_PHYS(r13)
1202 li r0, 0xff
1203 li r6, XICS_QIRR
1204 stbcix r0, r5, r6
1205
1206 /* increment the nap count and then go to nap mode */
1207 ld r4, HSTATE_KVM_VCORE(r13)
1208 addi r4, r4, VCORE_NAP_COUNT
1209 lwsync /* make previous updates visible */
121051: lwarx r3, 0, r4
1211 addi r3, r3, 1
1212 stwcx. r3, 0, r4
1213 bne 51b
1214 isync
1215
1216 mfspr r4, SPRN_LPCR
1217 li r0, LPCR_PECE
1218 andc r4, r4, r0
1219 ori r4, r4, LPCR_PECE0 /* exit nap on interrupt */
1220 mtspr SPRN_LPCR, r4
1221 li r0, 0
1222 std r0, HSTATE_SCRATCH0(r13)
1223 ptesync
1224 ld r0, HSTATE_SCRATCH0(r13)
12251: cmpd r0, r0
1226 bne 1b
1227 nap
1228 b .
1229
1230/*
1231 * Save away FP, VMX and VSX registers.
1232 * r3 = vcpu pointer
1233 */
1234_GLOBAL(kvmppc_save_fp)
1235 mfmsr r9
1236 ori r8,r9,MSR_FP
1237#ifdef CONFIG_ALTIVEC
1238BEGIN_FTR_SECTION
1239 oris r8,r8,MSR_VEC@h
1240END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
1241#endif
1242#ifdef CONFIG_VSX
1243BEGIN_FTR_SECTION
1244 oris r8,r8,MSR_VSX@h
1245END_FTR_SECTION_IFSET(CPU_FTR_VSX)
1246#endif
1247 mtmsrd r8
1248 isync
1249#ifdef CONFIG_VSX
1250BEGIN_FTR_SECTION
1251 reg = 0
1252 .rept 32
1253 li r6,reg*16+VCPU_VSRS
1254 stxvd2x reg,r6,r3
1255 reg = reg + 1
1256 .endr
1257FTR_SECTION_ELSE
1258#endif
1259 reg = 0
1260 .rept 32
1261 stfd reg,reg*8+VCPU_FPRS(r3)
1262 reg = reg + 1
1263 .endr
1264#ifdef CONFIG_VSX
1265ALT_FTR_SECTION_END_IFSET(CPU_FTR_VSX)
1266#endif
1267 mffs fr0
1268 stfd fr0,VCPU_FPSCR(r3)
1269
1270#ifdef CONFIG_ALTIVEC
1271BEGIN_FTR_SECTION
1272 reg = 0
1273 .rept 32
1274 li r6,reg*16+VCPU_VRS
1275 stvx reg,r6,r3
1276 reg = reg + 1
1277 .endr
1278 mfvscr vr0
1279 li r6,VCPU_VSCR
1280 stvx vr0,r6,r3
1281END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
1282#endif
1283 mfspr r6,SPRN_VRSAVE
1284 stw r6,VCPU_VRSAVE(r3)
1285 mtmsrd r9
1286 isync
1287 blr
1288
1289/*
1290 * Load up FP, VMX and VSX registers
1291 * r4 = vcpu pointer
1292 */
1293 .globl kvmppc_load_fp
1294kvmppc_load_fp:
1295 mfmsr r9
1296 ori r8,r9,MSR_FP
1297#ifdef CONFIG_ALTIVEC
1298BEGIN_FTR_SECTION
1299 oris r8,r8,MSR_VEC@h
1300END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
1301#endif
1302#ifdef CONFIG_VSX
1303BEGIN_FTR_SECTION
1304 oris r8,r8,MSR_VSX@h
1305END_FTR_SECTION_IFSET(CPU_FTR_VSX)
1306#endif
1307 mtmsrd r8
1308 isync
1309 lfd fr0,VCPU_FPSCR(r4)
1310 MTFSF_L(fr0)
1311#ifdef CONFIG_VSX
1312BEGIN_FTR_SECTION
1313 reg = 0
1314 .rept 32
1315 li r7,reg*16+VCPU_VSRS
1316 lxvd2x reg,r7,r4
1317 reg = reg + 1
1318 .endr
1319FTR_SECTION_ELSE
1320#endif
1321 reg = 0
1322 .rept 32
1323 lfd reg,reg*8+VCPU_FPRS(r4)
1324 reg = reg + 1
1325 .endr
1326#ifdef CONFIG_VSX
1327ALT_FTR_SECTION_END_IFSET(CPU_FTR_VSX)
1328#endif
1329
1330#ifdef CONFIG_ALTIVEC
1331BEGIN_FTR_SECTION
1332 li r7,VCPU_VSCR
1333 lvx vr0,r7,r4
1334 mtvscr vr0
1335 reg = 0
1336 .rept 32
1337 li r7,reg*16+VCPU_VRS
1338 lvx reg,r7,r4
1339 reg = reg + 1
1340 .endr
1341END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
1342#endif
1343 lwz r7,VCPU_VRSAVE(r4)
1344 mtspr SPRN_VRSAVE,r7
1345 blr
diff --git a/arch/powerpc/kvm/book3s_interrupts.S b/arch/powerpc/kvm/book3s_interrupts.S
index 2f0bc928b08a..c54b0e30cf3f 100644
--- a/arch/powerpc/kvm/book3s_interrupts.S
+++ b/arch/powerpc/kvm/book3s_interrupts.S
@@ -29,8 +29,7 @@
29#define ULONG_SIZE 8 29#define ULONG_SIZE 8
30#define FUNC(name) GLUE(.,name) 30#define FUNC(name) GLUE(.,name)
31 31
32#define GET_SHADOW_VCPU(reg) \ 32#define GET_SHADOW_VCPU_R13
33 addi reg, r13, PACA_KVM_SVCPU
34 33
35#define DISABLE_INTERRUPTS \ 34#define DISABLE_INTERRUPTS \
36 mfmsr r0; \ 35 mfmsr r0; \
@@ -43,8 +42,8 @@
43#define ULONG_SIZE 4 42#define ULONG_SIZE 4
44#define FUNC(name) name 43#define FUNC(name) name
45 44
46#define GET_SHADOW_VCPU(reg) \ 45#define GET_SHADOW_VCPU_R13 \
47 lwz reg, (THREAD + THREAD_KVM_SVCPU)(r2) 46 lwz r13, (THREAD + THREAD_KVM_SVCPU)(r2)
48 47
49#define DISABLE_INTERRUPTS \ 48#define DISABLE_INTERRUPTS \
50 mfmsr r0; \ 49 mfmsr r0; \
@@ -85,7 +84,7 @@
85 * r3: kvm_run pointer 84 * r3: kvm_run pointer
86 * r4: vcpu pointer 85 * r4: vcpu pointer
87 */ 86 */
88_GLOBAL(__kvmppc_vcpu_entry) 87_GLOBAL(__kvmppc_vcpu_run)
89 88
90kvm_start_entry: 89kvm_start_entry:
91 /* Write correct stack frame */ 90 /* Write correct stack frame */
@@ -107,17 +106,11 @@ kvm_start_entry:
107 /* Load non-volatile guest state from the vcpu */ 106 /* Load non-volatile guest state from the vcpu */
108 VCPU_LOAD_NVGPRS(r4) 107 VCPU_LOAD_NVGPRS(r4)
109 108
110 GET_SHADOW_VCPU(r5) 109kvm_start_lightweight:
111
112 /* Save R1/R2 in the PACA */
113 PPC_STL r1, SVCPU_HOST_R1(r5)
114 PPC_STL r2, SVCPU_HOST_R2(r5)
115 110
116 /* XXX swap in/out on load? */ 111 GET_SHADOW_VCPU_R13
117 PPC_LL r3, VCPU_HIGHMEM_HANDLER(r4) 112 PPC_LL r3, VCPU_HIGHMEM_HANDLER(r4)
118 PPC_STL r3, SVCPU_VMHANDLER(r5) 113 PPC_STL r3, HSTATE_VMHANDLER(r13)
119
120kvm_start_lightweight:
121 114
122 PPC_LL r10, VCPU_SHADOW_MSR(r4) /* r10 = vcpu->arch.shadow_msr */ 115 PPC_LL r10, VCPU_SHADOW_MSR(r4) /* r10 = vcpu->arch.shadow_msr */
123 116
diff --git a/arch/powerpc/kvm/book3s_mmu_hpte.c b/arch/powerpc/kvm/book3s_mmu_hpte.c
index 79751d8dd131..41cb0017e757 100644
--- a/arch/powerpc/kvm/book3s_mmu_hpte.c
+++ b/arch/powerpc/kvm/book3s_mmu_hpte.c
@@ -21,7 +21,6 @@
21#include <linux/kvm_host.h> 21#include <linux/kvm_host.h>
22#include <linux/hash.h> 22#include <linux/hash.h>
23#include <linux/slab.h> 23#include <linux/slab.h>
24#include "trace.h"
25 24
26#include <asm/kvm_ppc.h> 25#include <asm/kvm_ppc.h>
27#include <asm/kvm_book3s.h> 26#include <asm/kvm_book3s.h>
@@ -29,6 +28,8 @@
29#include <asm/mmu_context.h> 28#include <asm/mmu_context.h>
30#include <asm/hw_irq.h> 29#include <asm/hw_irq.h>
31 30
31#include "trace.h"
32
32#define PTE_SIZE 12 33#define PTE_SIZE 12
33 34
34static struct kmem_cache *hpte_cache; 35static struct kmem_cache *hpte_cache;
@@ -58,30 +59,31 @@ static inline u64 kvmppc_mmu_hash_vpte_long(u64 vpage)
58void kvmppc_mmu_hpte_cache_map(struct kvm_vcpu *vcpu, struct hpte_cache *pte) 59void kvmppc_mmu_hpte_cache_map(struct kvm_vcpu *vcpu, struct hpte_cache *pte)
59{ 60{
60 u64 index; 61 u64 index;
62 struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu);
61 63
62 trace_kvm_book3s_mmu_map(pte); 64 trace_kvm_book3s_mmu_map(pte);
63 65
64 spin_lock(&vcpu->arch.mmu_lock); 66 spin_lock(&vcpu3s->mmu_lock);
65 67
66 /* Add to ePTE list */ 68 /* Add to ePTE list */
67 index = kvmppc_mmu_hash_pte(pte->pte.eaddr); 69 index = kvmppc_mmu_hash_pte(pte->pte.eaddr);
68 hlist_add_head_rcu(&pte->list_pte, &vcpu->arch.hpte_hash_pte[index]); 70 hlist_add_head_rcu(&pte->list_pte, &vcpu3s->hpte_hash_pte[index]);
69 71
70 /* Add to ePTE_long list */ 72 /* Add to ePTE_long list */
71 index = kvmppc_mmu_hash_pte_long(pte->pte.eaddr); 73 index = kvmppc_mmu_hash_pte_long(pte->pte.eaddr);
72 hlist_add_head_rcu(&pte->list_pte_long, 74 hlist_add_head_rcu(&pte->list_pte_long,
73 &vcpu->arch.hpte_hash_pte_long[index]); 75 &vcpu3s->hpte_hash_pte_long[index]);
74 76
75 /* Add to vPTE list */ 77 /* Add to vPTE list */
76 index = kvmppc_mmu_hash_vpte(pte->pte.vpage); 78 index = kvmppc_mmu_hash_vpte(pte->pte.vpage);
77 hlist_add_head_rcu(&pte->list_vpte, &vcpu->arch.hpte_hash_vpte[index]); 79 hlist_add_head_rcu(&pte->list_vpte, &vcpu3s->hpte_hash_vpte[index]);
78 80
79 /* Add to vPTE_long list */ 81 /* Add to vPTE_long list */
80 index = kvmppc_mmu_hash_vpte_long(pte->pte.vpage); 82 index = kvmppc_mmu_hash_vpte_long(pte->pte.vpage);
81 hlist_add_head_rcu(&pte->list_vpte_long, 83 hlist_add_head_rcu(&pte->list_vpte_long,
82 &vcpu->arch.hpte_hash_vpte_long[index]); 84 &vcpu3s->hpte_hash_vpte_long[index]);
83 85
84 spin_unlock(&vcpu->arch.mmu_lock); 86 spin_unlock(&vcpu3s->mmu_lock);
85} 87}
86 88
87static void free_pte_rcu(struct rcu_head *head) 89static void free_pte_rcu(struct rcu_head *head)
@@ -92,16 +94,18 @@ static void free_pte_rcu(struct rcu_head *head)
92 94
93static void invalidate_pte(struct kvm_vcpu *vcpu, struct hpte_cache *pte) 95static void invalidate_pte(struct kvm_vcpu *vcpu, struct hpte_cache *pte)
94{ 96{
97 struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu);
98
95 trace_kvm_book3s_mmu_invalidate(pte); 99 trace_kvm_book3s_mmu_invalidate(pte);
96 100
97 /* Different for 32 and 64 bit */ 101 /* Different for 32 and 64 bit */
98 kvmppc_mmu_invalidate_pte(vcpu, pte); 102 kvmppc_mmu_invalidate_pte(vcpu, pte);
99 103
100 spin_lock(&vcpu->arch.mmu_lock); 104 spin_lock(&vcpu3s->mmu_lock);
101 105
102 /* pte already invalidated in between? */ 106 /* pte already invalidated in between? */
103 if (hlist_unhashed(&pte->list_pte)) { 107 if (hlist_unhashed(&pte->list_pte)) {
104 spin_unlock(&vcpu->arch.mmu_lock); 108 spin_unlock(&vcpu3s->mmu_lock);
105 return; 109 return;
106 } 110 }
107 111
@@ -115,14 +119,15 @@ static void invalidate_pte(struct kvm_vcpu *vcpu, struct hpte_cache *pte)
115 else 119 else
116 kvm_release_pfn_clean(pte->pfn); 120 kvm_release_pfn_clean(pte->pfn);
117 121
118 spin_unlock(&vcpu->arch.mmu_lock); 122 spin_unlock(&vcpu3s->mmu_lock);
119 123
120 vcpu->arch.hpte_cache_count--; 124 vcpu3s->hpte_cache_count--;
121 call_rcu(&pte->rcu_head, free_pte_rcu); 125 call_rcu(&pte->rcu_head, free_pte_rcu);
122} 126}
123 127
124static void kvmppc_mmu_pte_flush_all(struct kvm_vcpu *vcpu) 128static void kvmppc_mmu_pte_flush_all(struct kvm_vcpu *vcpu)
125{ 129{
130 struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu);
126 struct hpte_cache *pte; 131 struct hpte_cache *pte;
127 struct hlist_node *node; 132 struct hlist_node *node;
128 int i; 133 int i;
@@ -130,7 +135,7 @@ static void kvmppc_mmu_pte_flush_all(struct kvm_vcpu *vcpu)
130 rcu_read_lock(); 135 rcu_read_lock();
131 136
132 for (i = 0; i < HPTEG_HASH_NUM_VPTE_LONG; i++) { 137 for (i = 0; i < HPTEG_HASH_NUM_VPTE_LONG; i++) {
133 struct hlist_head *list = &vcpu->arch.hpte_hash_vpte_long[i]; 138 struct hlist_head *list = &vcpu3s->hpte_hash_vpte_long[i];
134 139
135 hlist_for_each_entry_rcu(pte, node, list, list_vpte_long) 140 hlist_for_each_entry_rcu(pte, node, list, list_vpte_long)
136 invalidate_pte(vcpu, pte); 141 invalidate_pte(vcpu, pte);
@@ -141,12 +146,13 @@ static void kvmppc_mmu_pte_flush_all(struct kvm_vcpu *vcpu)
141 146
142static void kvmppc_mmu_pte_flush_page(struct kvm_vcpu *vcpu, ulong guest_ea) 147static void kvmppc_mmu_pte_flush_page(struct kvm_vcpu *vcpu, ulong guest_ea)
143{ 148{
149 struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu);
144 struct hlist_head *list; 150 struct hlist_head *list;
145 struct hlist_node *node; 151 struct hlist_node *node;
146 struct hpte_cache *pte; 152 struct hpte_cache *pte;
147 153
148 /* Find the list of entries in the map */ 154 /* Find the list of entries in the map */
149 list = &vcpu->arch.hpte_hash_pte[kvmppc_mmu_hash_pte(guest_ea)]; 155 list = &vcpu3s->hpte_hash_pte[kvmppc_mmu_hash_pte(guest_ea)];
150 156
151 rcu_read_lock(); 157 rcu_read_lock();
152 158
@@ -160,12 +166,13 @@ static void kvmppc_mmu_pte_flush_page(struct kvm_vcpu *vcpu, ulong guest_ea)
160 166
161static void kvmppc_mmu_pte_flush_long(struct kvm_vcpu *vcpu, ulong guest_ea) 167static void kvmppc_mmu_pte_flush_long(struct kvm_vcpu *vcpu, ulong guest_ea)
162{ 168{
169 struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu);
163 struct hlist_head *list; 170 struct hlist_head *list;
164 struct hlist_node *node; 171 struct hlist_node *node;
165 struct hpte_cache *pte; 172 struct hpte_cache *pte;
166 173
167 /* Find the list of entries in the map */ 174 /* Find the list of entries in the map */
168 list = &vcpu->arch.hpte_hash_pte_long[ 175 list = &vcpu3s->hpte_hash_pte_long[
169 kvmppc_mmu_hash_pte_long(guest_ea)]; 176 kvmppc_mmu_hash_pte_long(guest_ea)];
170 177
171 rcu_read_lock(); 178 rcu_read_lock();
@@ -203,12 +210,13 @@ void kvmppc_mmu_pte_flush(struct kvm_vcpu *vcpu, ulong guest_ea, ulong ea_mask)
203/* Flush with mask 0xfffffffff */ 210/* Flush with mask 0xfffffffff */
204static void kvmppc_mmu_pte_vflush_short(struct kvm_vcpu *vcpu, u64 guest_vp) 211static void kvmppc_mmu_pte_vflush_short(struct kvm_vcpu *vcpu, u64 guest_vp)
205{ 212{
213 struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu);
206 struct hlist_head *list; 214 struct hlist_head *list;
207 struct hlist_node *node; 215 struct hlist_node *node;
208 struct hpte_cache *pte; 216 struct hpte_cache *pte;
209 u64 vp_mask = 0xfffffffffULL; 217 u64 vp_mask = 0xfffffffffULL;
210 218
211 list = &vcpu->arch.hpte_hash_vpte[kvmppc_mmu_hash_vpte(guest_vp)]; 219 list = &vcpu3s->hpte_hash_vpte[kvmppc_mmu_hash_vpte(guest_vp)];
212 220
213 rcu_read_lock(); 221 rcu_read_lock();
214 222
@@ -223,12 +231,13 @@ static void kvmppc_mmu_pte_vflush_short(struct kvm_vcpu *vcpu, u64 guest_vp)
223/* Flush with mask 0xffffff000 */ 231/* Flush with mask 0xffffff000 */
224static void kvmppc_mmu_pte_vflush_long(struct kvm_vcpu *vcpu, u64 guest_vp) 232static void kvmppc_mmu_pte_vflush_long(struct kvm_vcpu *vcpu, u64 guest_vp)
225{ 233{
234 struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu);
226 struct hlist_head *list; 235 struct hlist_head *list;
227 struct hlist_node *node; 236 struct hlist_node *node;
228 struct hpte_cache *pte; 237 struct hpte_cache *pte;
229 u64 vp_mask = 0xffffff000ULL; 238 u64 vp_mask = 0xffffff000ULL;
230 239
231 list = &vcpu->arch.hpte_hash_vpte_long[ 240 list = &vcpu3s->hpte_hash_vpte_long[
232 kvmppc_mmu_hash_vpte_long(guest_vp)]; 241 kvmppc_mmu_hash_vpte_long(guest_vp)];
233 242
234 rcu_read_lock(); 243 rcu_read_lock();
@@ -261,6 +270,7 @@ void kvmppc_mmu_pte_vflush(struct kvm_vcpu *vcpu, u64 guest_vp, u64 vp_mask)
261 270
262void kvmppc_mmu_pte_pflush(struct kvm_vcpu *vcpu, ulong pa_start, ulong pa_end) 271void kvmppc_mmu_pte_pflush(struct kvm_vcpu *vcpu, ulong pa_start, ulong pa_end)
263{ 272{
273 struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu);
264 struct hlist_node *node; 274 struct hlist_node *node;
265 struct hpte_cache *pte; 275 struct hpte_cache *pte;
266 int i; 276 int i;
@@ -270,7 +280,7 @@ void kvmppc_mmu_pte_pflush(struct kvm_vcpu *vcpu, ulong pa_start, ulong pa_end)
270 rcu_read_lock(); 280 rcu_read_lock();
271 281
272 for (i = 0; i < HPTEG_HASH_NUM_VPTE_LONG; i++) { 282 for (i = 0; i < HPTEG_HASH_NUM_VPTE_LONG; i++) {
273 struct hlist_head *list = &vcpu->arch.hpte_hash_vpte_long[i]; 283 struct hlist_head *list = &vcpu3s->hpte_hash_vpte_long[i];
274 284
275 hlist_for_each_entry_rcu(pte, node, list, list_vpte_long) 285 hlist_for_each_entry_rcu(pte, node, list, list_vpte_long)
276 if ((pte->pte.raddr >= pa_start) && 286 if ((pte->pte.raddr >= pa_start) &&
@@ -283,12 +293,13 @@ void kvmppc_mmu_pte_pflush(struct kvm_vcpu *vcpu, ulong pa_start, ulong pa_end)
283 293
284struct hpte_cache *kvmppc_mmu_hpte_cache_next(struct kvm_vcpu *vcpu) 294struct hpte_cache *kvmppc_mmu_hpte_cache_next(struct kvm_vcpu *vcpu)
285{ 295{
296 struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu);
286 struct hpte_cache *pte; 297 struct hpte_cache *pte;
287 298
288 pte = kmem_cache_zalloc(hpte_cache, GFP_KERNEL); 299 pte = kmem_cache_zalloc(hpte_cache, GFP_KERNEL);
289 vcpu->arch.hpte_cache_count++; 300 vcpu3s->hpte_cache_count++;
290 301
291 if (vcpu->arch.hpte_cache_count == HPTEG_CACHE_NUM) 302 if (vcpu3s->hpte_cache_count == HPTEG_CACHE_NUM)
292 kvmppc_mmu_pte_flush_all(vcpu); 303 kvmppc_mmu_pte_flush_all(vcpu);
293 304
294 return pte; 305 return pte;
@@ -309,17 +320,19 @@ static void kvmppc_mmu_hpte_init_hash(struct hlist_head *hash_list, int len)
309 320
310int kvmppc_mmu_hpte_init(struct kvm_vcpu *vcpu) 321int kvmppc_mmu_hpte_init(struct kvm_vcpu *vcpu)
311{ 322{
323 struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu);
324
312 /* init hpte lookup hashes */ 325 /* init hpte lookup hashes */
313 kvmppc_mmu_hpte_init_hash(vcpu->arch.hpte_hash_pte, 326 kvmppc_mmu_hpte_init_hash(vcpu3s->hpte_hash_pte,
314 ARRAY_SIZE(vcpu->arch.hpte_hash_pte)); 327 ARRAY_SIZE(vcpu3s->hpte_hash_pte));
315 kvmppc_mmu_hpte_init_hash(vcpu->arch.hpte_hash_pte_long, 328 kvmppc_mmu_hpte_init_hash(vcpu3s->hpte_hash_pte_long,
316 ARRAY_SIZE(vcpu->arch.hpte_hash_pte_long)); 329 ARRAY_SIZE(vcpu3s->hpte_hash_pte_long));
317 kvmppc_mmu_hpte_init_hash(vcpu->arch.hpte_hash_vpte, 330 kvmppc_mmu_hpte_init_hash(vcpu3s->hpte_hash_vpte,
318 ARRAY_SIZE(vcpu->arch.hpte_hash_vpte)); 331 ARRAY_SIZE(vcpu3s->hpte_hash_vpte));
319 kvmppc_mmu_hpte_init_hash(vcpu->arch.hpte_hash_vpte_long, 332 kvmppc_mmu_hpte_init_hash(vcpu3s->hpte_hash_vpte_long,
320 ARRAY_SIZE(vcpu->arch.hpte_hash_vpte_long)); 333 ARRAY_SIZE(vcpu3s->hpte_hash_vpte_long));
321 334
322 spin_lock_init(&vcpu->arch.mmu_lock); 335 spin_lock_init(&vcpu3s->mmu_lock);
323 336
324 return 0; 337 return 0;
325} 338}
diff --git a/arch/powerpc/kvm/book3s_pr.c b/arch/powerpc/kvm/book3s_pr.c
new file mode 100644
index 000000000000..0c0d3f274437
--- /dev/null
+++ b/arch/powerpc/kvm/book3s_pr.c
@@ -0,0 +1,1029 @@
1/*
2 * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved.
3 *
4 * Authors:
5 * Alexander Graf <agraf@suse.de>
6 * Kevin Wolf <mail@kevin-wolf.de>
7 * Paul Mackerras <paulus@samba.org>
8 *
9 * Description:
10 * Functions relating to running KVM on Book 3S processors where
11 * we don't have access to hypervisor mode, and we run the guest
12 * in problem state (user mode).
13 *
14 * This file is derived from arch/powerpc/kvm/44x.c,
15 * by Hollis Blanchard <hollisb@us.ibm.com>.
16 *
17 * This program is free software; you can redistribute it and/or modify
18 * it under the terms of the GNU General Public License, version 2, as
19 * published by the Free Software Foundation.
20 */
21
22#include <linux/kvm_host.h>
23#include <linux/err.h>
24#include <linux/slab.h>
25
26#include <asm/reg.h>
27#include <asm/cputable.h>
28#include <asm/cacheflush.h>
29#include <asm/tlbflush.h>
30#include <asm/uaccess.h>
31#include <asm/io.h>
32#include <asm/kvm_ppc.h>
33#include <asm/kvm_book3s.h>
34#include <asm/mmu_context.h>
35#include <linux/gfp.h>
36#include <linux/sched.h>
37#include <linux/vmalloc.h>
38#include <linux/highmem.h>
39
40#include "trace.h"
41
42/* #define EXIT_DEBUG */
43/* #define DEBUG_EXT */
44
45static int kvmppc_handle_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr,
46 ulong msr);
47
48/* Some compatibility defines */
49#ifdef CONFIG_PPC_BOOK3S_32
50#define MSR_USER32 MSR_USER
51#define MSR_USER64 MSR_USER
52#define HW_PAGE_SIZE PAGE_SIZE
53#endif
54
55void kvmppc_core_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
56{
57#ifdef CONFIG_PPC_BOOK3S_64
58 memcpy(to_svcpu(vcpu)->slb, to_book3s(vcpu)->slb_shadow, sizeof(to_svcpu(vcpu)->slb));
59 memcpy(&get_paca()->shadow_vcpu, to_book3s(vcpu)->shadow_vcpu,
60 sizeof(get_paca()->shadow_vcpu));
61 to_svcpu(vcpu)->slb_max = to_book3s(vcpu)->slb_shadow_max;
62#endif
63
64#ifdef CONFIG_PPC_BOOK3S_32
65 current->thread.kvm_shadow_vcpu = to_book3s(vcpu)->shadow_vcpu;
66#endif
67}
68
69void kvmppc_core_vcpu_put(struct kvm_vcpu *vcpu)
70{
71#ifdef CONFIG_PPC_BOOK3S_64
72 memcpy(to_book3s(vcpu)->slb_shadow, to_svcpu(vcpu)->slb, sizeof(to_svcpu(vcpu)->slb));
73 memcpy(to_book3s(vcpu)->shadow_vcpu, &get_paca()->shadow_vcpu,
74 sizeof(get_paca()->shadow_vcpu));
75 to_book3s(vcpu)->slb_shadow_max = to_svcpu(vcpu)->slb_max;
76#endif
77
78 kvmppc_giveup_ext(vcpu, MSR_FP);
79 kvmppc_giveup_ext(vcpu, MSR_VEC);
80 kvmppc_giveup_ext(vcpu, MSR_VSX);
81}
82
83static void kvmppc_recalc_shadow_msr(struct kvm_vcpu *vcpu)
84{
85 ulong smsr = vcpu->arch.shared->msr;
86
87 /* Guest MSR values */
88 smsr &= MSR_FE0 | MSR_FE1 | MSR_SF | MSR_SE | MSR_BE | MSR_DE;
89 /* Process MSR values */
90 smsr |= MSR_ME | MSR_RI | MSR_IR | MSR_DR | MSR_PR | MSR_EE;
91 /* External providers the guest reserved */
92 smsr |= (vcpu->arch.shared->msr & vcpu->arch.guest_owned_ext);
93 /* 64-bit Process MSR values */
94#ifdef CONFIG_PPC_BOOK3S_64
95 smsr |= MSR_ISF | MSR_HV;
96#endif
97 vcpu->arch.shadow_msr = smsr;
98}
99
100void kvmppc_set_msr(struct kvm_vcpu *vcpu, u64 msr)
101{
102 ulong old_msr = vcpu->arch.shared->msr;
103
104#ifdef EXIT_DEBUG
105 printk(KERN_INFO "KVM: Set MSR to 0x%llx\n", msr);
106#endif
107
108 msr &= to_book3s(vcpu)->msr_mask;
109 vcpu->arch.shared->msr = msr;
110 kvmppc_recalc_shadow_msr(vcpu);
111
112 if (msr & MSR_POW) {
113 if (!vcpu->arch.pending_exceptions) {
114 kvm_vcpu_block(vcpu);
115 vcpu->stat.halt_wakeup++;
116
117 /* Unset POW bit after we woke up */
118 msr &= ~MSR_POW;
119 vcpu->arch.shared->msr = msr;
120 }
121 }
122
123 if ((vcpu->arch.shared->msr & (MSR_PR|MSR_IR|MSR_DR)) !=
124 (old_msr & (MSR_PR|MSR_IR|MSR_DR))) {
125 kvmppc_mmu_flush_segments(vcpu);
126 kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu));
127
128 /* Preload magic page segment when in kernel mode */
129 if (!(msr & MSR_PR) && vcpu->arch.magic_page_pa) {
130 struct kvm_vcpu_arch *a = &vcpu->arch;
131
132 if (msr & MSR_DR)
133 kvmppc_mmu_map_segment(vcpu, a->magic_page_ea);
134 else
135 kvmppc_mmu_map_segment(vcpu, a->magic_page_pa);
136 }
137 }
138
139 /* Preload FPU if it's enabled */
140 if (vcpu->arch.shared->msr & MSR_FP)
141 kvmppc_handle_ext(vcpu, BOOK3S_INTERRUPT_FP_UNAVAIL, MSR_FP);
142}
143
144void kvmppc_set_pvr(struct kvm_vcpu *vcpu, u32 pvr)
145{
146 u32 host_pvr;
147
148 vcpu->arch.hflags &= ~BOOK3S_HFLAG_SLB;
149 vcpu->arch.pvr = pvr;
150#ifdef CONFIG_PPC_BOOK3S_64
151 if ((pvr >= 0x330000) && (pvr < 0x70330000)) {
152 kvmppc_mmu_book3s_64_init(vcpu);
153 to_book3s(vcpu)->hior = 0xfff00000;
154 to_book3s(vcpu)->msr_mask = 0xffffffffffffffffULL;
155 } else
156#endif
157 {
158 kvmppc_mmu_book3s_32_init(vcpu);
159 to_book3s(vcpu)->hior = 0;
160 to_book3s(vcpu)->msr_mask = 0xffffffffULL;
161 }
162
163 /* If we are in hypervisor level on 970, we can tell the CPU to
164 * treat DCBZ as 32 bytes store */
165 vcpu->arch.hflags &= ~BOOK3S_HFLAG_DCBZ32;
166 if (vcpu->arch.mmu.is_dcbz32(vcpu) && (mfmsr() & MSR_HV) &&
167 !strcmp(cur_cpu_spec->platform, "ppc970"))
168 vcpu->arch.hflags |= BOOK3S_HFLAG_DCBZ32;
169
170 /* Cell performs badly if MSR_FEx are set. So let's hope nobody
171 really needs them in a VM on Cell and force disable them. */
172 if (!strcmp(cur_cpu_spec->platform, "ppc-cell-be"))
173 to_book3s(vcpu)->msr_mask &= ~(MSR_FE0 | MSR_FE1);
174
175#ifdef CONFIG_PPC_BOOK3S_32
176 /* 32 bit Book3S always has 32 byte dcbz */
177 vcpu->arch.hflags |= BOOK3S_HFLAG_DCBZ32;
178#endif
179
180 /* On some CPUs we can execute paired single operations natively */
181 asm ( "mfpvr %0" : "=r"(host_pvr));
182 switch (host_pvr) {
183 case 0x00080200: /* lonestar 2.0 */
184 case 0x00088202: /* lonestar 2.2 */
185 case 0x70000100: /* gekko 1.0 */
186 case 0x00080100: /* gekko 2.0 */
187 case 0x00083203: /* gekko 2.3a */
188 case 0x00083213: /* gekko 2.3b */
189 case 0x00083204: /* gekko 2.4 */
190 case 0x00083214: /* gekko 2.4e (8SE) - retail HW2 */
191 case 0x00087200: /* broadway */
192 vcpu->arch.hflags |= BOOK3S_HFLAG_NATIVE_PS;
193 /* Enable HID2.PSE - in case we need it later */
194 mtspr(SPRN_HID2_GEKKO, mfspr(SPRN_HID2_GEKKO) | (1 << 29));
195 }
196}
197
198/* Book3s_32 CPUs always have 32 bytes cache line size, which Linux assumes. To
199 * make Book3s_32 Linux work on Book3s_64, we have to make sure we trap dcbz to
200 * emulate 32 bytes dcbz length.
201 *
202 * The Book3s_64 inventors also realized this case and implemented a special bit
203 * in the HID5 register, which is a hypervisor ressource. Thus we can't use it.
204 *
205 * My approach here is to patch the dcbz instruction on executing pages.
206 */
207static void kvmppc_patch_dcbz(struct kvm_vcpu *vcpu, struct kvmppc_pte *pte)
208{
209 struct page *hpage;
210 u64 hpage_offset;
211 u32 *page;
212 int i;
213
214 hpage = gfn_to_page(vcpu->kvm, pte->raddr >> PAGE_SHIFT);
215 if (is_error_page(hpage)) {
216 kvm_release_page_clean(hpage);
217 return;
218 }
219
220 hpage_offset = pte->raddr & ~PAGE_MASK;
221 hpage_offset &= ~0xFFFULL;
222 hpage_offset /= 4;
223
224 get_page(hpage);
225 page = kmap_atomic(hpage, KM_USER0);
226
227 /* patch dcbz into reserved instruction, so we trap */
228 for (i=hpage_offset; i < hpage_offset + (HW_PAGE_SIZE / 4); i++)
229 if ((page[i] & 0xff0007ff) == INS_DCBZ)
230 page[i] &= 0xfffffff7;
231
232 kunmap_atomic(page, KM_USER0);
233 put_page(hpage);
234}
235
236static int kvmppc_visible_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
237{
238 ulong mp_pa = vcpu->arch.magic_page_pa;
239
240 if (unlikely(mp_pa) &&
241 unlikely((mp_pa & KVM_PAM) >> PAGE_SHIFT == gfn)) {
242 return 1;
243 }
244
245 return kvm_is_visible_gfn(vcpu->kvm, gfn);
246}
247
248int kvmppc_handle_pagefault(struct kvm_run *run, struct kvm_vcpu *vcpu,
249 ulong eaddr, int vec)
250{
251 bool data = (vec == BOOK3S_INTERRUPT_DATA_STORAGE);
252 int r = RESUME_GUEST;
253 int relocated;
254 int page_found = 0;
255 struct kvmppc_pte pte;
256 bool is_mmio = false;
257 bool dr = (vcpu->arch.shared->msr & MSR_DR) ? true : false;
258 bool ir = (vcpu->arch.shared->msr & MSR_IR) ? true : false;
259 u64 vsid;
260
261 relocated = data ? dr : ir;
262
263 /* Resolve real address if translation turned on */
264 if (relocated) {
265 page_found = vcpu->arch.mmu.xlate(vcpu, eaddr, &pte, data);
266 } else {
267 pte.may_execute = true;
268 pte.may_read = true;
269 pte.may_write = true;
270 pte.raddr = eaddr & KVM_PAM;
271 pte.eaddr = eaddr;
272 pte.vpage = eaddr >> 12;
273 }
274
275 switch (vcpu->arch.shared->msr & (MSR_DR|MSR_IR)) {
276 case 0:
277 pte.vpage |= ((u64)VSID_REAL << (SID_SHIFT - 12));
278 break;
279 case MSR_DR:
280 case MSR_IR:
281 vcpu->arch.mmu.esid_to_vsid(vcpu, eaddr >> SID_SHIFT, &vsid);
282
283 if ((vcpu->arch.shared->msr & (MSR_DR|MSR_IR)) == MSR_DR)
284 pte.vpage |= ((u64)VSID_REAL_DR << (SID_SHIFT - 12));
285 else
286 pte.vpage |= ((u64)VSID_REAL_IR << (SID_SHIFT - 12));
287 pte.vpage |= vsid;
288
289 if (vsid == -1)
290 page_found = -EINVAL;
291 break;
292 }
293
294 if (vcpu->arch.mmu.is_dcbz32(vcpu) &&
295 (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32))) {
296 /*
297 * If we do the dcbz hack, we have to NX on every execution,
298 * so we can patch the executing code. This renders our guest
299 * NX-less.
300 */
301 pte.may_execute = !data;
302 }
303
304 if (page_found == -ENOENT) {
305 /* Page not found in guest PTE entries */
306 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu);
307 vcpu->arch.shared->dsisr = to_svcpu(vcpu)->fault_dsisr;
308 vcpu->arch.shared->msr |=
309 (to_svcpu(vcpu)->shadow_srr1 & 0x00000000f8000000ULL);
310 kvmppc_book3s_queue_irqprio(vcpu, vec);
311 } else if (page_found == -EPERM) {
312 /* Storage protection */
313 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu);
314 vcpu->arch.shared->dsisr =
315 to_svcpu(vcpu)->fault_dsisr & ~DSISR_NOHPTE;
316 vcpu->arch.shared->dsisr |= DSISR_PROTFAULT;
317 vcpu->arch.shared->msr |=
318 (to_svcpu(vcpu)->shadow_srr1 & 0x00000000f8000000ULL);
319 kvmppc_book3s_queue_irqprio(vcpu, vec);
320 } else if (page_found == -EINVAL) {
321 /* Page not found in guest SLB */
322 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu);
323 kvmppc_book3s_queue_irqprio(vcpu, vec + 0x80);
324 } else if (!is_mmio &&
325 kvmppc_visible_gfn(vcpu, pte.raddr >> PAGE_SHIFT)) {
326 /* The guest's PTE is not mapped yet. Map on the host */
327 kvmppc_mmu_map_page(vcpu, &pte);
328 if (data)
329 vcpu->stat.sp_storage++;
330 else if (vcpu->arch.mmu.is_dcbz32(vcpu) &&
331 (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32)))
332 kvmppc_patch_dcbz(vcpu, &pte);
333 } else {
334 /* MMIO */
335 vcpu->stat.mmio_exits++;
336 vcpu->arch.paddr_accessed = pte.raddr;
337 r = kvmppc_emulate_mmio(run, vcpu);
338 if ( r == RESUME_HOST_NV )
339 r = RESUME_HOST;
340 }
341
342 return r;
343}
344
345static inline int get_fpr_index(int i)
346{
347#ifdef CONFIG_VSX
348 i *= 2;
349#endif
350 return i;
351}
352
353/* Give up external provider (FPU, Altivec, VSX) */
354void kvmppc_giveup_ext(struct kvm_vcpu *vcpu, ulong msr)
355{
356 struct thread_struct *t = &current->thread;
357 u64 *vcpu_fpr = vcpu->arch.fpr;
358#ifdef CONFIG_VSX
359 u64 *vcpu_vsx = vcpu->arch.vsr;
360#endif
361 u64 *thread_fpr = (u64*)t->fpr;
362 int i;
363
364 if (!(vcpu->arch.guest_owned_ext & msr))
365 return;
366
367#ifdef DEBUG_EXT
368 printk(KERN_INFO "Giving up ext 0x%lx\n", msr);
369#endif
370
371 switch (msr) {
372 case MSR_FP:
373 giveup_fpu(current);
374 for (i = 0; i < ARRAY_SIZE(vcpu->arch.fpr); i++)
375 vcpu_fpr[i] = thread_fpr[get_fpr_index(i)];
376
377 vcpu->arch.fpscr = t->fpscr.val;
378 break;
379 case MSR_VEC:
380#ifdef CONFIG_ALTIVEC
381 giveup_altivec(current);
382 memcpy(vcpu->arch.vr, t->vr, sizeof(vcpu->arch.vr));
383 vcpu->arch.vscr = t->vscr;
384#endif
385 break;
386 case MSR_VSX:
387#ifdef CONFIG_VSX
388 __giveup_vsx(current);
389 for (i = 0; i < ARRAY_SIZE(vcpu->arch.vsr); i++)
390 vcpu_vsx[i] = thread_fpr[get_fpr_index(i) + 1];
391#endif
392 break;
393 default:
394 BUG();
395 }
396
397 vcpu->arch.guest_owned_ext &= ~msr;
398 current->thread.regs->msr &= ~msr;
399 kvmppc_recalc_shadow_msr(vcpu);
400}
401
402static int kvmppc_read_inst(struct kvm_vcpu *vcpu)
403{
404 ulong srr0 = kvmppc_get_pc(vcpu);
405 u32 last_inst = kvmppc_get_last_inst(vcpu);
406 int ret;
407
408 ret = kvmppc_ld(vcpu, &srr0, sizeof(u32), &last_inst, false);
409 if (ret == -ENOENT) {
410 ulong msr = vcpu->arch.shared->msr;
411
412 msr = kvmppc_set_field(msr, 33, 33, 1);
413 msr = kvmppc_set_field(msr, 34, 36, 0);
414 vcpu->arch.shared->msr = kvmppc_set_field(msr, 42, 47, 0);
415 kvmppc_book3s_queue_irqprio(vcpu, BOOK3S_INTERRUPT_INST_STORAGE);
416 return EMULATE_AGAIN;
417 }
418
419 return EMULATE_DONE;
420}
421
422static int kvmppc_check_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr)
423{
424
425 /* Need to do paired single emulation? */
426 if (!(vcpu->arch.hflags & BOOK3S_HFLAG_PAIRED_SINGLE))
427 return EMULATE_DONE;
428
429 /* Read out the instruction */
430 if (kvmppc_read_inst(vcpu) == EMULATE_DONE)
431 /* Need to emulate */
432 return EMULATE_FAIL;
433
434 return EMULATE_AGAIN;
435}
436
437/* Handle external providers (FPU, Altivec, VSX) */
438static int kvmppc_handle_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr,
439 ulong msr)
440{
441 struct thread_struct *t = &current->thread;
442 u64 *vcpu_fpr = vcpu->arch.fpr;
443#ifdef CONFIG_VSX
444 u64 *vcpu_vsx = vcpu->arch.vsr;
445#endif
446 u64 *thread_fpr = (u64*)t->fpr;
447 int i;
448
449 /* When we have paired singles, we emulate in software */
450 if (vcpu->arch.hflags & BOOK3S_HFLAG_PAIRED_SINGLE)
451 return RESUME_GUEST;
452
453 if (!(vcpu->arch.shared->msr & msr)) {
454 kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
455 return RESUME_GUEST;
456 }
457
458 /* We already own the ext */
459 if (vcpu->arch.guest_owned_ext & msr) {
460 return RESUME_GUEST;
461 }
462
463#ifdef DEBUG_EXT
464 printk(KERN_INFO "Loading up ext 0x%lx\n", msr);
465#endif
466
467 current->thread.regs->msr |= msr;
468
469 switch (msr) {
470 case MSR_FP:
471 for (i = 0; i < ARRAY_SIZE(vcpu->arch.fpr); i++)
472 thread_fpr[get_fpr_index(i)] = vcpu_fpr[i];
473
474 t->fpscr.val = vcpu->arch.fpscr;
475 t->fpexc_mode = 0;
476 kvmppc_load_up_fpu();
477 break;
478 case MSR_VEC:
479#ifdef CONFIG_ALTIVEC
480 memcpy(t->vr, vcpu->arch.vr, sizeof(vcpu->arch.vr));
481 t->vscr = vcpu->arch.vscr;
482 t->vrsave = -1;
483 kvmppc_load_up_altivec();
484#endif
485 break;
486 case MSR_VSX:
487#ifdef CONFIG_VSX
488 for (i = 0; i < ARRAY_SIZE(vcpu->arch.vsr); i++)
489 thread_fpr[get_fpr_index(i) + 1] = vcpu_vsx[i];
490 kvmppc_load_up_vsx();
491#endif
492 break;
493 default:
494 BUG();
495 }
496
497 vcpu->arch.guest_owned_ext |= msr;
498
499 kvmppc_recalc_shadow_msr(vcpu);
500
501 return RESUME_GUEST;
502}
503
504int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu,
505 unsigned int exit_nr)
506{
507 int r = RESUME_HOST;
508
509 vcpu->stat.sum_exits++;
510
511 run->exit_reason = KVM_EXIT_UNKNOWN;
512 run->ready_for_interrupt_injection = 1;
513
514 trace_kvm_book3s_exit(exit_nr, vcpu);
515 kvm_resched(vcpu);
516 switch (exit_nr) {
517 case BOOK3S_INTERRUPT_INST_STORAGE:
518 vcpu->stat.pf_instruc++;
519
520#ifdef CONFIG_PPC_BOOK3S_32
521 /* We set segments as unused segments when invalidating them. So
522 * treat the respective fault as segment fault. */
523 if (to_svcpu(vcpu)->sr[kvmppc_get_pc(vcpu) >> SID_SHIFT]
524 == SR_INVALID) {
525 kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu));
526 r = RESUME_GUEST;
527 break;
528 }
529#endif
530
531 /* only care about PTEG not found errors, but leave NX alone */
532 if (to_svcpu(vcpu)->shadow_srr1 & 0x40000000) {
533 r = kvmppc_handle_pagefault(run, vcpu, kvmppc_get_pc(vcpu), exit_nr);
534 vcpu->stat.sp_instruc++;
535 } else if (vcpu->arch.mmu.is_dcbz32(vcpu) &&
536 (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32))) {
537 /*
538 * XXX If we do the dcbz hack we use the NX bit to flush&patch the page,
539 * so we can't use the NX bit inside the guest. Let's cross our fingers,
540 * that no guest that needs the dcbz hack does NX.
541 */
542 kvmppc_mmu_pte_flush(vcpu, kvmppc_get_pc(vcpu), ~0xFFFUL);
543 r = RESUME_GUEST;
544 } else {
545 vcpu->arch.shared->msr |=
546 to_svcpu(vcpu)->shadow_srr1 & 0x58000000;
547 kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
548 r = RESUME_GUEST;
549 }
550 break;
551 case BOOK3S_INTERRUPT_DATA_STORAGE:
552 {
553 ulong dar = kvmppc_get_fault_dar(vcpu);
554 vcpu->stat.pf_storage++;
555
556#ifdef CONFIG_PPC_BOOK3S_32
557 /* We set segments as unused segments when invalidating them. So
558 * treat the respective fault as segment fault. */
559 if ((to_svcpu(vcpu)->sr[dar >> SID_SHIFT]) == SR_INVALID) {
560 kvmppc_mmu_map_segment(vcpu, dar);
561 r = RESUME_GUEST;
562 break;
563 }
564#endif
565
566 /* The only case we need to handle is missing shadow PTEs */
567 if (to_svcpu(vcpu)->fault_dsisr & DSISR_NOHPTE) {
568 r = kvmppc_handle_pagefault(run, vcpu, dar, exit_nr);
569 } else {
570 vcpu->arch.shared->dar = dar;
571 vcpu->arch.shared->dsisr = to_svcpu(vcpu)->fault_dsisr;
572 kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
573 r = RESUME_GUEST;
574 }
575 break;
576 }
577 case BOOK3S_INTERRUPT_DATA_SEGMENT:
578 if (kvmppc_mmu_map_segment(vcpu, kvmppc_get_fault_dar(vcpu)) < 0) {
579 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu);
580 kvmppc_book3s_queue_irqprio(vcpu,
581 BOOK3S_INTERRUPT_DATA_SEGMENT);
582 }
583 r = RESUME_GUEST;
584 break;
585 case BOOK3S_INTERRUPT_INST_SEGMENT:
586 if (kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu)) < 0) {
587 kvmppc_book3s_queue_irqprio(vcpu,
588 BOOK3S_INTERRUPT_INST_SEGMENT);
589 }
590 r = RESUME_GUEST;
591 break;
592 /* We're good on these - the host merely wanted to get our attention */
593 case BOOK3S_INTERRUPT_DECREMENTER:
594 vcpu->stat.dec_exits++;
595 r = RESUME_GUEST;
596 break;
597 case BOOK3S_INTERRUPT_EXTERNAL:
598 vcpu->stat.ext_intr_exits++;
599 r = RESUME_GUEST;
600 break;
601 case BOOK3S_INTERRUPT_PERFMON:
602 r = RESUME_GUEST;
603 break;
604 case BOOK3S_INTERRUPT_PROGRAM:
605 {
606 enum emulation_result er;
607 ulong flags;
608
609program_interrupt:
610 flags = to_svcpu(vcpu)->shadow_srr1 & 0x1f0000ull;
611
612 if (vcpu->arch.shared->msr & MSR_PR) {
613#ifdef EXIT_DEBUG
614 printk(KERN_INFO "Userspace triggered 0x700 exception at 0x%lx (0x%x)\n", kvmppc_get_pc(vcpu), kvmppc_get_last_inst(vcpu));
615#endif
616 if ((kvmppc_get_last_inst(vcpu) & 0xff0007ff) !=
617 (INS_DCBZ & 0xfffffff7)) {
618 kvmppc_core_queue_program(vcpu, flags);
619 r = RESUME_GUEST;
620 break;
621 }
622 }
623
624 vcpu->stat.emulated_inst_exits++;
625 er = kvmppc_emulate_instruction(run, vcpu);
626 switch (er) {
627 case EMULATE_DONE:
628 r = RESUME_GUEST_NV;
629 break;
630 case EMULATE_AGAIN:
631 r = RESUME_GUEST;
632 break;
633 case EMULATE_FAIL:
634 printk(KERN_CRIT "%s: emulation at %lx failed (%08x)\n",
635 __func__, kvmppc_get_pc(vcpu), kvmppc_get_last_inst(vcpu));
636 kvmppc_core_queue_program(vcpu, flags);
637 r = RESUME_GUEST;
638 break;
639 case EMULATE_DO_MMIO:
640 run->exit_reason = KVM_EXIT_MMIO;
641 r = RESUME_HOST_NV;
642 break;
643 default:
644 BUG();
645 }
646 break;
647 }
648 case BOOK3S_INTERRUPT_SYSCALL:
649 if (vcpu->arch.osi_enabled &&
650 (((u32)kvmppc_get_gpr(vcpu, 3)) == OSI_SC_MAGIC_R3) &&
651 (((u32)kvmppc_get_gpr(vcpu, 4)) == OSI_SC_MAGIC_R4)) {
652 /* MOL hypercalls */
653 u64 *gprs = run->osi.gprs;
654 int i;
655
656 run->exit_reason = KVM_EXIT_OSI;
657 for (i = 0; i < 32; i++)
658 gprs[i] = kvmppc_get_gpr(vcpu, i);
659 vcpu->arch.osi_needed = 1;
660 r = RESUME_HOST_NV;
661 } else if (!(vcpu->arch.shared->msr & MSR_PR) &&
662 (((u32)kvmppc_get_gpr(vcpu, 0)) == KVM_SC_MAGIC_R0)) {
663 /* KVM PV hypercalls */
664 kvmppc_set_gpr(vcpu, 3, kvmppc_kvm_pv(vcpu));
665 r = RESUME_GUEST;
666 } else {
667 /* Guest syscalls */
668 vcpu->stat.syscall_exits++;
669 kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
670 r = RESUME_GUEST;
671 }
672 break;
673 case BOOK3S_INTERRUPT_FP_UNAVAIL:
674 case BOOK3S_INTERRUPT_ALTIVEC:
675 case BOOK3S_INTERRUPT_VSX:
676 {
677 int ext_msr = 0;
678
679 switch (exit_nr) {
680 case BOOK3S_INTERRUPT_FP_UNAVAIL: ext_msr = MSR_FP; break;
681 case BOOK3S_INTERRUPT_ALTIVEC: ext_msr = MSR_VEC; break;
682 case BOOK3S_INTERRUPT_VSX: ext_msr = MSR_VSX; break;
683 }
684
685 switch (kvmppc_check_ext(vcpu, exit_nr)) {
686 case EMULATE_DONE:
687 /* everything ok - let's enable the ext */
688 r = kvmppc_handle_ext(vcpu, exit_nr, ext_msr);
689 break;
690 case EMULATE_FAIL:
691 /* we need to emulate this instruction */
692 goto program_interrupt;
693 break;
694 default:
695 /* nothing to worry about - go again */
696 break;
697 }
698 break;
699 }
700 case BOOK3S_INTERRUPT_ALIGNMENT:
701 if (kvmppc_read_inst(vcpu) == EMULATE_DONE) {
702 vcpu->arch.shared->dsisr = kvmppc_alignment_dsisr(vcpu,
703 kvmppc_get_last_inst(vcpu));
704 vcpu->arch.shared->dar = kvmppc_alignment_dar(vcpu,
705 kvmppc_get_last_inst(vcpu));
706 kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
707 }
708 r = RESUME_GUEST;
709 break;
710 case BOOK3S_INTERRUPT_MACHINE_CHECK:
711 case BOOK3S_INTERRUPT_TRACE:
712 kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
713 r = RESUME_GUEST;
714 break;
715 default:
716 /* Ugh - bork here! What did we get? */
717 printk(KERN_EMERG "exit_nr=0x%x | pc=0x%lx | msr=0x%lx\n",
718 exit_nr, kvmppc_get_pc(vcpu), to_svcpu(vcpu)->shadow_srr1);
719 r = RESUME_HOST;
720 BUG();
721 break;
722 }
723
724
725 if (!(r & RESUME_HOST)) {
726 /* To avoid clobbering exit_reason, only check for signals if
727 * we aren't already exiting to userspace for some other
728 * reason. */
729 if (signal_pending(current)) {
730#ifdef EXIT_DEBUG
731 printk(KERN_EMERG "KVM: Going back to host\n");
732#endif
733 vcpu->stat.signal_exits++;
734 run->exit_reason = KVM_EXIT_INTR;
735 r = -EINTR;
736 } else {
737 /* In case an interrupt came in that was triggered
738 * from userspace (like DEC), we need to check what
739 * to inject now! */
740 kvmppc_core_deliver_interrupts(vcpu);
741 }
742 }
743
744 trace_kvm_book3s_reenter(r, vcpu);
745
746 return r;
747}
748
749int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
750 struct kvm_sregs *sregs)
751{
752 struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu);
753 int i;
754
755 sregs->pvr = vcpu->arch.pvr;
756
757 sregs->u.s.sdr1 = to_book3s(vcpu)->sdr1;
758 if (vcpu->arch.hflags & BOOK3S_HFLAG_SLB) {
759 for (i = 0; i < 64; i++) {
760 sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige | i;
761 sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv;
762 }
763 } else {
764 for (i = 0; i < 16; i++)
765 sregs->u.s.ppc32.sr[i] = vcpu->arch.shared->sr[i];
766
767 for (i = 0; i < 8; i++) {
768 sregs->u.s.ppc32.ibat[i] = vcpu3s->ibat[i].raw;
769 sregs->u.s.ppc32.dbat[i] = vcpu3s->dbat[i].raw;
770 }
771 }
772
773 return 0;
774}
775
776int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
777 struct kvm_sregs *sregs)
778{
779 struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu);
780 int i;
781
782 kvmppc_set_pvr(vcpu, sregs->pvr);
783
784 vcpu3s->sdr1 = sregs->u.s.sdr1;
785 if (vcpu->arch.hflags & BOOK3S_HFLAG_SLB) {
786 for (i = 0; i < 64; i++) {
787 vcpu->arch.mmu.slbmte(vcpu, sregs->u.s.ppc64.slb[i].slbv,
788 sregs->u.s.ppc64.slb[i].slbe);
789 }
790 } else {
791 for (i = 0; i < 16; i++) {
792 vcpu->arch.mmu.mtsrin(vcpu, i, sregs->u.s.ppc32.sr[i]);
793 }
794 for (i = 0; i < 8; i++) {
795 kvmppc_set_bat(vcpu, &(vcpu3s->ibat[i]), false,
796 (u32)sregs->u.s.ppc32.ibat[i]);
797 kvmppc_set_bat(vcpu, &(vcpu3s->ibat[i]), true,
798 (u32)(sregs->u.s.ppc32.ibat[i] >> 32));
799 kvmppc_set_bat(vcpu, &(vcpu3s->dbat[i]), false,
800 (u32)sregs->u.s.ppc32.dbat[i]);
801 kvmppc_set_bat(vcpu, &(vcpu3s->dbat[i]), true,
802 (u32)(sregs->u.s.ppc32.dbat[i] >> 32));
803 }
804 }
805
806 /* Flush the MMU after messing with the segments */
807 kvmppc_mmu_pte_flush(vcpu, 0, 0);
808
809 return 0;
810}
811
812int kvmppc_core_check_processor_compat(void)
813{
814 return 0;
815}
816
817struct kvm_vcpu *kvmppc_core_vcpu_create(struct kvm *kvm, unsigned int id)
818{
819 struct kvmppc_vcpu_book3s *vcpu_book3s;
820 struct kvm_vcpu *vcpu;
821 int err = -ENOMEM;
822 unsigned long p;
823
824 vcpu_book3s = vzalloc(sizeof(struct kvmppc_vcpu_book3s));
825 if (!vcpu_book3s)
826 goto out;
827
828 vcpu_book3s->shadow_vcpu = (struct kvmppc_book3s_shadow_vcpu *)
829 kzalloc(sizeof(*vcpu_book3s->shadow_vcpu), GFP_KERNEL);
830 if (!vcpu_book3s->shadow_vcpu)
831 goto free_vcpu;
832
833 vcpu = &vcpu_book3s->vcpu;
834 err = kvm_vcpu_init(vcpu, kvm, id);
835 if (err)
836 goto free_shadow_vcpu;
837
838 p = __get_free_page(GFP_KERNEL|__GFP_ZERO);
839 /* the real shared page fills the last 4k of our page */
840 vcpu->arch.shared = (void*)(p + PAGE_SIZE - 4096);
841 if (!p)
842 goto uninit_vcpu;
843
844 vcpu->arch.host_retip = kvm_return_point;
845 vcpu->arch.host_msr = mfmsr();
846#ifdef CONFIG_PPC_BOOK3S_64
847 /* default to book3s_64 (970fx) */
848 vcpu->arch.pvr = 0x3C0301;
849#else
850 /* default to book3s_32 (750) */
851 vcpu->arch.pvr = 0x84202;
852#endif
853 kvmppc_set_pvr(vcpu, vcpu->arch.pvr);
854 vcpu->arch.slb_nr = 64;
855
856 /* remember where some real-mode handlers are */
857 vcpu->arch.trampoline_lowmem = __pa(kvmppc_handler_lowmem_trampoline);
858 vcpu->arch.trampoline_enter = __pa(kvmppc_handler_trampoline_enter);
859 vcpu->arch.highmem_handler = (ulong)kvmppc_handler_highmem;
860#ifdef CONFIG_PPC_BOOK3S_64
861 vcpu->arch.rmcall = *(ulong*)kvmppc_rmcall;
862#else
863 vcpu->arch.rmcall = (ulong)kvmppc_rmcall;
864#endif
865
866 vcpu->arch.shadow_msr = MSR_USER64;
867
868 err = kvmppc_mmu_init(vcpu);
869 if (err < 0)
870 goto uninit_vcpu;
871
872 return vcpu;
873
874uninit_vcpu:
875 kvm_vcpu_uninit(vcpu);
876free_shadow_vcpu:
877 kfree(vcpu_book3s->shadow_vcpu);
878free_vcpu:
879 vfree(vcpu_book3s);
880out:
881 return ERR_PTR(err);
882}
883
884void kvmppc_core_vcpu_free(struct kvm_vcpu *vcpu)
885{
886 struct kvmppc_vcpu_book3s *vcpu_book3s = to_book3s(vcpu);
887
888 free_page((unsigned long)vcpu->arch.shared & PAGE_MASK);
889 kvm_vcpu_uninit(vcpu);
890 kfree(vcpu_book3s->shadow_vcpu);
891 vfree(vcpu_book3s);
892}
893
894int kvmppc_vcpu_run(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
895{
896 int ret;
897 double fpr[32][TS_FPRWIDTH];
898 unsigned int fpscr;
899 int fpexc_mode;
900#ifdef CONFIG_ALTIVEC
901 vector128 vr[32];
902 vector128 vscr;
903 unsigned long uninitialized_var(vrsave);
904 int used_vr;
905#endif
906#ifdef CONFIG_VSX
907 int used_vsr;
908#endif
909 ulong ext_msr;
910
911 /* No need to go into the guest when all we do is going out */
912 if (signal_pending(current)) {
913 kvm_run->exit_reason = KVM_EXIT_INTR;
914 return -EINTR;
915 }
916
917 /* Save FPU state in stack */
918 if (current->thread.regs->msr & MSR_FP)
919 giveup_fpu(current);
920 memcpy(fpr, current->thread.fpr, sizeof(current->thread.fpr));
921 fpscr = current->thread.fpscr.val;
922 fpexc_mode = current->thread.fpexc_mode;
923
924#ifdef CONFIG_ALTIVEC
925 /* Save Altivec state in stack */
926 used_vr = current->thread.used_vr;
927 if (used_vr) {
928 if (current->thread.regs->msr & MSR_VEC)
929 giveup_altivec(current);
930 memcpy(vr, current->thread.vr, sizeof(current->thread.vr));
931 vscr = current->thread.vscr;
932 vrsave = current->thread.vrsave;
933 }
934#endif
935
936#ifdef CONFIG_VSX
937 /* Save VSX state in stack */
938 used_vsr = current->thread.used_vsr;
939 if (used_vsr && (current->thread.regs->msr & MSR_VSX))
940 __giveup_vsx(current);
941#endif
942
943 /* Remember the MSR with disabled extensions */
944 ext_msr = current->thread.regs->msr;
945
946 /* Preload FPU if it's enabled */
947 if (vcpu->arch.shared->msr & MSR_FP)
948 kvmppc_handle_ext(vcpu, BOOK3S_INTERRUPT_FP_UNAVAIL, MSR_FP);
949
950 kvm_guest_enter();
951
952 ret = __kvmppc_vcpu_run(kvm_run, vcpu);
953
954 kvm_guest_exit();
955
956 local_irq_disable();
957
958 current->thread.regs->msr = ext_msr;
959
960 /* Make sure we save the guest FPU/Altivec/VSX state */
961 kvmppc_giveup_ext(vcpu, MSR_FP);
962 kvmppc_giveup_ext(vcpu, MSR_VEC);
963 kvmppc_giveup_ext(vcpu, MSR_VSX);
964
965 /* Restore FPU state from stack */
966 memcpy(current->thread.fpr, fpr, sizeof(current->thread.fpr));
967 current->thread.fpscr.val = fpscr;
968 current->thread.fpexc_mode = fpexc_mode;
969
970#ifdef CONFIG_ALTIVEC
971 /* Restore Altivec state from stack */
972 if (used_vr && current->thread.used_vr) {
973 memcpy(current->thread.vr, vr, sizeof(current->thread.vr));
974 current->thread.vscr = vscr;
975 current->thread.vrsave = vrsave;
976 }
977 current->thread.used_vr = used_vr;
978#endif
979
980#ifdef CONFIG_VSX
981 current->thread.used_vsr = used_vsr;
982#endif
983
984 return ret;
985}
986
987int kvmppc_core_prepare_memory_region(struct kvm *kvm,
988 struct kvm_userspace_memory_region *mem)
989{
990 return 0;
991}
992
993void kvmppc_core_commit_memory_region(struct kvm *kvm,
994 struct kvm_userspace_memory_region *mem)
995{
996}
997
998int kvmppc_core_init_vm(struct kvm *kvm)
999{
1000 return 0;
1001}
1002
1003void kvmppc_core_destroy_vm(struct kvm *kvm)
1004{
1005}
1006
1007static int kvmppc_book3s_init(void)
1008{
1009 int r;
1010
1011 r = kvm_init(NULL, sizeof(struct kvmppc_vcpu_book3s), 0,
1012 THIS_MODULE);
1013
1014 if (r)
1015 return r;
1016
1017 r = kvmppc_mmu_hpte_sysinit();
1018
1019 return r;
1020}
1021
1022static void kvmppc_book3s_exit(void)
1023{
1024 kvmppc_mmu_hpte_sysexit();
1025 kvm_exit();
1026}
1027
1028module_init(kvmppc_book3s_init);
1029module_exit(kvmppc_book3s_exit);
diff --git a/arch/powerpc/kvm/book3s_rmhandlers.S b/arch/powerpc/kvm/book3s_rmhandlers.S
index 1a1b34487e71..c1f877c4a884 100644
--- a/arch/powerpc/kvm/book3s_rmhandlers.S
+++ b/arch/powerpc/kvm/book3s_rmhandlers.S
@@ -36,41 +36,44 @@
36#if defined(CONFIG_PPC_BOOK3S_64) 36#if defined(CONFIG_PPC_BOOK3S_64)
37 37
38#define LOAD_SHADOW_VCPU(reg) GET_PACA(reg) 38#define LOAD_SHADOW_VCPU(reg) GET_PACA(reg)
39#define SHADOW_VCPU_OFF PACA_KVM_SVCPU
40#define MSR_NOIRQ MSR_KERNEL & ~(MSR_IR | MSR_DR) 39#define MSR_NOIRQ MSR_KERNEL & ~(MSR_IR | MSR_DR)
41#define FUNC(name) GLUE(.,name) 40#define FUNC(name) GLUE(.,name)
42 41
42kvmppc_skip_interrupt:
43 /*
44 * Here all GPRs are unchanged from when the interrupt happened
45 * except for r13, which is saved in SPRG_SCRATCH0.
46 */
47 mfspr r13, SPRN_SRR0
48 addi r13, r13, 4
49 mtspr SPRN_SRR0, r13
50 GET_SCRATCH0(r13)
51 rfid
52 b .
53
54kvmppc_skip_Hinterrupt:
55 /*
56 * Here all GPRs are unchanged from when the interrupt happened
57 * except for r13, which is saved in SPRG_SCRATCH0.
58 */
59 mfspr r13, SPRN_HSRR0
60 addi r13, r13, 4
61 mtspr SPRN_HSRR0, r13
62 GET_SCRATCH0(r13)
63 hrfid
64 b .
65
43#elif defined(CONFIG_PPC_BOOK3S_32) 66#elif defined(CONFIG_PPC_BOOK3S_32)
44 67
45#define LOAD_SHADOW_VCPU(reg) \
46 mfspr reg, SPRN_SPRG_THREAD; \
47 lwz reg, THREAD_KVM_SVCPU(reg); \
48 /* PPC32 can have a NULL pointer - let's check for that */ \
49 mtspr SPRN_SPRG_SCRATCH1, r12; /* Save r12 */ \
50 mfcr r12; \
51 cmpwi reg, 0; \
52 bne 1f; \
53 mfspr reg, SPRN_SPRG_SCRATCH0; \
54 mtcr r12; \
55 mfspr r12, SPRN_SPRG_SCRATCH1; \
56 b kvmppc_resume_\intno; \
571:; \
58 mtcr r12; \
59 mfspr r12, SPRN_SPRG_SCRATCH1; \
60 tophys(reg, reg)
61
62#define SHADOW_VCPU_OFF 0
63#define MSR_NOIRQ MSR_KERNEL 68#define MSR_NOIRQ MSR_KERNEL
64#define FUNC(name) name 69#define FUNC(name) name
65 70
66#endif
67
68.macro INTERRUPT_TRAMPOLINE intno 71.macro INTERRUPT_TRAMPOLINE intno
69 72
70.global kvmppc_trampoline_\intno 73.global kvmppc_trampoline_\intno
71kvmppc_trampoline_\intno: 74kvmppc_trampoline_\intno:
72 75
73 SET_SCRATCH0(r13) /* Save r13 */ 76 mtspr SPRN_SPRG_SCRATCH0, r13 /* Save r13 */
74 77
75 /* 78 /*
76 * First thing to do is to find out if we're coming 79 * First thing to do is to find out if we're coming
@@ -78,19 +81,28 @@ kvmppc_trampoline_\intno:
78 * 81 *
79 * To distinguish, we check a magic byte in the PACA/current 82 * To distinguish, we check a magic byte in the PACA/current
80 */ 83 */
81 LOAD_SHADOW_VCPU(r13) 84 mfspr r13, SPRN_SPRG_THREAD
82 PPC_STL r12, (SHADOW_VCPU_OFF + SVCPU_SCRATCH0)(r13) 85 lwz r13, THREAD_KVM_SVCPU(r13)
86 /* PPC32 can have a NULL pointer - let's check for that */
87 mtspr SPRN_SPRG_SCRATCH1, r12 /* Save r12 */
83 mfcr r12 88 mfcr r12
84 stw r12, (SHADOW_VCPU_OFF + SVCPU_SCRATCH1)(r13) 89 cmpwi r13, 0
85 lbz r12, (SHADOW_VCPU_OFF + SVCPU_IN_GUEST)(r13) 90 bne 1f
912: mtcr r12
92 mfspr r12, SPRN_SPRG_SCRATCH1
93 mfspr r13, SPRN_SPRG_SCRATCH0 /* r13 = original r13 */
94 b kvmppc_resume_\intno /* Get back original handler */
95
961: tophys(r13, r13)
97 stw r12, HSTATE_SCRATCH1(r13)
98 mfspr r12, SPRN_SPRG_SCRATCH1
99 stw r12, HSTATE_SCRATCH0(r13)
100 lbz r12, HSTATE_IN_GUEST(r13)
86 cmpwi r12, KVM_GUEST_MODE_NONE 101 cmpwi r12, KVM_GUEST_MODE_NONE
87 bne ..kvmppc_handler_hasmagic_\intno 102 bne ..kvmppc_handler_hasmagic_\intno
88 /* No KVM guest? Then jump back to the Linux handler! */ 103 /* No KVM guest? Then jump back to the Linux handler! */
89 lwz r12, (SHADOW_VCPU_OFF + SVCPU_SCRATCH1)(r13) 104 lwz r12, HSTATE_SCRATCH1(r13)
90 mtcr r12 105 b 2b
91 PPC_LL r12, (SHADOW_VCPU_OFF + SVCPU_SCRATCH0)(r13)
92 GET_SCRATCH0(r13) /* r13 = original r13 */
93 b kvmppc_resume_\intno /* Get back original handler */
94 106
95 /* Now we know we're handling a KVM guest */ 107 /* Now we know we're handling a KVM guest */
96..kvmppc_handler_hasmagic_\intno: 108..kvmppc_handler_hasmagic_\intno:
@@ -112,9 +124,6 @@ INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_MACHINE_CHECK
112INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_DATA_STORAGE 124INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_DATA_STORAGE
113INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_INST_STORAGE 125INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_INST_STORAGE
114INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_EXTERNAL 126INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_EXTERNAL
115#ifdef CONFIG_PPC_BOOK3S_64
116INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_EXTERNAL_HV
117#endif
118INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_ALIGNMENT 127INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_ALIGNMENT
119INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_PROGRAM 128INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_PROGRAM
120INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_FP_UNAVAIL 129INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_FP_UNAVAIL
@@ -124,14 +133,6 @@ INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_TRACE
124INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_PERFMON 133INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_PERFMON
125INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_ALTIVEC 134INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_ALTIVEC
126 135
127/* Those are only available on 64 bit machines */
128
129#ifdef CONFIG_PPC_BOOK3S_64
130INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_DATA_SEGMENT
131INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_INST_SEGMENT
132INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_VSX
133#endif
134
135/* 136/*
136 * Bring us back to the faulting code, but skip the 137 * Bring us back to the faulting code, but skip the
137 * faulting instruction. 138 * faulting instruction.
@@ -143,8 +144,8 @@ INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_VSX
143 * 144 *
144 * R12 = free 145 * R12 = free
145 * R13 = Shadow VCPU (PACA) 146 * R13 = Shadow VCPU (PACA)
146 * SVCPU.SCRATCH0 = guest R12 147 * HSTATE.SCRATCH0 = guest R12
147 * SVCPU.SCRATCH1 = guest CR 148 * HSTATE.SCRATCH1 = guest CR
148 * SPRG_SCRATCH0 = guest R13 149 * SPRG_SCRATCH0 = guest R13
149 * 150 *
150 */ 151 */
@@ -156,13 +157,14 @@ kvmppc_handler_skip_ins:
156 mtsrr0 r12 157 mtsrr0 r12
157 158
158 /* Clean up all state */ 159 /* Clean up all state */
159 lwz r12, (SHADOW_VCPU_OFF + SVCPU_SCRATCH1)(r13) 160 lwz r12, HSTATE_SCRATCH1(r13)
160 mtcr r12 161 mtcr r12
161 PPC_LL r12, (SHADOW_VCPU_OFF + SVCPU_SCRATCH0)(r13) 162 PPC_LL r12, HSTATE_SCRATCH0(r13)
162 GET_SCRATCH0(r13) 163 GET_SCRATCH0(r13)
163 164
164 /* And get back into the code */ 165 /* And get back into the code */
165 RFI 166 RFI
167#endif
166 168
167/* 169/*
168 * This trampoline brings us back to a real mode handler 170 * This trampoline brings us back to a real mode handler
@@ -251,12 +253,4 @@ define_load_up(altivec)
251define_load_up(vsx) 253define_load_up(vsx)
252#endif 254#endif
253 255
254.global kvmppc_trampoline_lowmem
255kvmppc_trampoline_lowmem:
256 PPC_LONG kvmppc_handler_lowmem_trampoline - CONFIG_KERNEL_START
257
258.global kvmppc_trampoline_enter
259kvmppc_trampoline_enter:
260 PPC_LONG kvmppc_handler_trampoline_enter - CONFIG_KERNEL_START
261
262#include "book3s_segment.S" 256#include "book3s_segment.S"
diff --git a/arch/powerpc/kvm/book3s_segment.S b/arch/powerpc/kvm/book3s_segment.S
index 451264274b8c..aed32e517212 100644
--- a/arch/powerpc/kvm/book3s_segment.S
+++ b/arch/powerpc/kvm/book3s_segment.S
@@ -22,7 +22,7 @@
22#if defined(CONFIG_PPC_BOOK3S_64) 22#if defined(CONFIG_PPC_BOOK3S_64)
23 23
24#define GET_SHADOW_VCPU(reg) \ 24#define GET_SHADOW_VCPU(reg) \
25 addi reg, r13, PACA_KVM_SVCPU 25 mr reg, r13
26 26
27#elif defined(CONFIG_PPC_BOOK3S_32) 27#elif defined(CONFIG_PPC_BOOK3S_32)
28 28
@@ -71,6 +71,10 @@ kvmppc_handler_trampoline_enter:
71 /* r3 = shadow vcpu */ 71 /* r3 = shadow vcpu */
72 GET_SHADOW_VCPU(r3) 72 GET_SHADOW_VCPU(r3)
73 73
74 /* Save R1/R2 in the PACA (64-bit) or shadow_vcpu (32-bit) */
75 PPC_STL r1, HSTATE_HOST_R1(r3)
76 PPC_STL r2, HSTATE_HOST_R2(r3)
77
74 /* Move SRR0 and SRR1 into the respective regs */ 78 /* Move SRR0 and SRR1 into the respective regs */
75 PPC_LL r9, SVCPU_PC(r3) 79 PPC_LL r9, SVCPU_PC(r3)
76 mtsrr0 r9 80 mtsrr0 r9
@@ -78,36 +82,36 @@ kvmppc_handler_trampoline_enter:
78 82
79 /* Activate guest mode, so faults get handled by KVM */ 83 /* Activate guest mode, so faults get handled by KVM */
80 li r11, KVM_GUEST_MODE_GUEST 84 li r11, KVM_GUEST_MODE_GUEST
81 stb r11, SVCPU_IN_GUEST(r3) 85 stb r11, HSTATE_IN_GUEST(r3)
82 86
83 /* Switch to guest segment. This is subarch specific. */ 87 /* Switch to guest segment. This is subarch specific. */
84 LOAD_GUEST_SEGMENTS 88 LOAD_GUEST_SEGMENTS
85 89
86 /* Enter guest */ 90 /* Enter guest */
87 91
88 PPC_LL r4, (SVCPU_CTR)(r3) 92 PPC_LL r4, SVCPU_CTR(r3)
89 PPC_LL r5, (SVCPU_LR)(r3) 93 PPC_LL r5, SVCPU_LR(r3)
90 lwz r6, (SVCPU_CR)(r3) 94 lwz r6, SVCPU_CR(r3)
91 lwz r7, (SVCPU_XER)(r3) 95 lwz r7, SVCPU_XER(r3)
92 96
93 mtctr r4 97 mtctr r4
94 mtlr r5 98 mtlr r5
95 mtcr r6 99 mtcr r6
96 mtxer r7 100 mtxer r7
97 101
98 PPC_LL r0, (SVCPU_R0)(r3) 102 PPC_LL r0, SVCPU_R0(r3)
99 PPC_LL r1, (SVCPU_R1)(r3) 103 PPC_LL r1, SVCPU_R1(r3)
100 PPC_LL r2, (SVCPU_R2)(r3) 104 PPC_LL r2, SVCPU_R2(r3)
101 PPC_LL r4, (SVCPU_R4)(r3) 105 PPC_LL r4, SVCPU_R4(r3)
102 PPC_LL r5, (SVCPU_R5)(r3) 106 PPC_LL r5, SVCPU_R5(r3)
103 PPC_LL r6, (SVCPU_R6)(r3) 107 PPC_LL r6, SVCPU_R6(r3)
104 PPC_LL r7, (SVCPU_R7)(r3) 108 PPC_LL r7, SVCPU_R7(r3)
105 PPC_LL r8, (SVCPU_R8)(r3) 109 PPC_LL r8, SVCPU_R8(r3)
106 PPC_LL r9, (SVCPU_R9)(r3) 110 PPC_LL r9, SVCPU_R9(r3)
107 PPC_LL r10, (SVCPU_R10)(r3) 111 PPC_LL r10, SVCPU_R10(r3)
108 PPC_LL r11, (SVCPU_R11)(r3) 112 PPC_LL r11, SVCPU_R11(r3)
109 PPC_LL r12, (SVCPU_R12)(r3) 113 PPC_LL r12, SVCPU_R12(r3)
110 PPC_LL r13, (SVCPU_R13)(r3) 114 PPC_LL r13, SVCPU_R13(r3)
111 115
112 PPC_LL r3, (SVCPU_R3)(r3) 116 PPC_LL r3, (SVCPU_R3)(r3)
113 117
@@ -125,56 +129,63 @@ kvmppc_handler_trampoline_enter_end:
125.global kvmppc_handler_trampoline_exit 129.global kvmppc_handler_trampoline_exit
126kvmppc_handler_trampoline_exit: 130kvmppc_handler_trampoline_exit:
127 131
132.global kvmppc_interrupt
133kvmppc_interrupt:
134
128 /* Register usage at this point: 135 /* Register usage at this point:
129 * 136 *
130 * SPRG_SCRATCH0 = guest R13 137 * SPRG_SCRATCH0 = guest R13
131 * R12 = exit handler id 138 * R12 = exit handler id
132 * R13 = shadow vcpu - SHADOW_VCPU_OFF [=PACA on PPC64] 139 * R13 = shadow vcpu (32-bit) or PACA (64-bit)
133 * SVCPU.SCRATCH0 = guest R12 140 * HSTATE.SCRATCH0 = guest R12
134 * SVCPU.SCRATCH1 = guest CR 141 * HSTATE.SCRATCH1 = guest CR
135 * 142 *
136 */ 143 */
137 144
138 /* Save registers */ 145 /* Save registers */
139 146
140 PPC_STL r0, (SHADOW_VCPU_OFF + SVCPU_R0)(r13) 147 PPC_STL r0, SVCPU_R0(r13)
141 PPC_STL r1, (SHADOW_VCPU_OFF + SVCPU_R1)(r13) 148 PPC_STL r1, SVCPU_R1(r13)
142 PPC_STL r2, (SHADOW_VCPU_OFF + SVCPU_R2)(r13) 149 PPC_STL r2, SVCPU_R2(r13)
143 PPC_STL r3, (SHADOW_VCPU_OFF + SVCPU_R3)(r13) 150 PPC_STL r3, SVCPU_R3(r13)
144 PPC_STL r4, (SHADOW_VCPU_OFF + SVCPU_R4)(r13) 151 PPC_STL r4, SVCPU_R4(r13)
145 PPC_STL r5, (SHADOW_VCPU_OFF + SVCPU_R5)(r13) 152 PPC_STL r5, SVCPU_R5(r13)
146 PPC_STL r6, (SHADOW_VCPU_OFF + SVCPU_R6)(r13) 153 PPC_STL r6, SVCPU_R6(r13)
147 PPC_STL r7, (SHADOW_VCPU_OFF + SVCPU_R7)(r13) 154 PPC_STL r7, SVCPU_R7(r13)
148 PPC_STL r8, (SHADOW_VCPU_OFF + SVCPU_R8)(r13) 155 PPC_STL r8, SVCPU_R8(r13)
149 PPC_STL r9, (SHADOW_VCPU_OFF + SVCPU_R9)(r13) 156 PPC_STL r9, SVCPU_R9(r13)
150 PPC_STL r10, (SHADOW_VCPU_OFF + SVCPU_R10)(r13) 157 PPC_STL r10, SVCPU_R10(r13)
151 PPC_STL r11, (SHADOW_VCPU_OFF + SVCPU_R11)(r13) 158 PPC_STL r11, SVCPU_R11(r13)
152 159
153 /* Restore R1/R2 so we can handle faults */ 160 /* Restore R1/R2 so we can handle faults */
154 PPC_LL r1, (SHADOW_VCPU_OFF + SVCPU_HOST_R1)(r13) 161 PPC_LL r1, HSTATE_HOST_R1(r13)
155 PPC_LL r2, (SHADOW_VCPU_OFF + SVCPU_HOST_R2)(r13) 162 PPC_LL r2, HSTATE_HOST_R2(r13)
156 163
157 /* Save guest PC and MSR */ 164 /* Save guest PC and MSR */
165#ifdef CONFIG_PPC64
166BEGIN_FTR_SECTION
158 andi. r0,r12,0x2 167 andi. r0,r12,0x2
159 beq 1f 168 beq 1f
160 mfspr r3,SPRN_HSRR0 169 mfspr r3,SPRN_HSRR0
161 mfspr r4,SPRN_HSRR1 170 mfspr r4,SPRN_HSRR1
162 andi. r12,r12,0x3ffd 171 andi. r12,r12,0x3ffd
163 b 2f 172 b 2f
173END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
174#endif
1641: mfsrr0 r3 1751: mfsrr0 r3
165 mfsrr1 r4 176 mfsrr1 r4
1662: 1772:
167 PPC_STL r3, (SHADOW_VCPU_OFF + SVCPU_PC)(r13) 178 PPC_STL r3, SVCPU_PC(r13)
168 PPC_STL r4, (SHADOW_VCPU_OFF + SVCPU_SHADOW_SRR1)(r13) 179 PPC_STL r4, SVCPU_SHADOW_SRR1(r13)
169 180
170 /* Get scratch'ed off registers */ 181 /* Get scratch'ed off registers */
171 GET_SCRATCH0(r9) 182 GET_SCRATCH0(r9)
172 PPC_LL r8, (SHADOW_VCPU_OFF + SVCPU_SCRATCH0)(r13) 183 PPC_LL r8, HSTATE_SCRATCH0(r13)
173 lwz r7, (SHADOW_VCPU_OFF + SVCPU_SCRATCH1)(r13) 184 lwz r7, HSTATE_SCRATCH1(r13)
174 185
175 PPC_STL r9, (SHADOW_VCPU_OFF + SVCPU_R13)(r13) 186 PPC_STL r9, SVCPU_R13(r13)
176 PPC_STL r8, (SHADOW_VCPU_OFF + SVCPU_R12)(r13) 187 PPC_STL r8, SVCPU_R12(r13)
177 stw r7, (SHADOW_VCPU_OFF + SVCPU_CR)(r13) 188 stw r7, SVCPU_CR(r13)
178 189
179 /* Save more register state */ 190 /* Save more register state */
180 191
@@ -184,11 +195,11 @@ kvmppc_handler_trampoline_exit:
184 mfctr r8 195 mfctr r8
185 mflr r9 196 mflr r9
186 197
187 stw r5, (SHADOW_VCPU_OFF + SVCPU_XER)(r13) 198 stw r5, SVCPU_XER(r13)
188 PPC_STL r6, (SHADOW_VCPU_OFF + SVCPU_FAULT_DAR)(r13) 199 PPC_STL r6, SVCPU_FAULT_DAR(r13)
189 stw r7, (SHADOW_VCPU_OFF + SVCPU_FAULT_DSISR)(r13) 200 stw r7, SVCPU_FAULT_DSISR(r13)
190 PPC_STL r8, (SHADOW_VCPU_OFF + SVCPU_CTR)(r13) 201 PPC_STL r8, SVCPU_CTR(r13)
191 PPC_STL r9, (SHADOW_VCPU_OFF + SVCPU_LR)(r13) 202 PPC_STL r9, SVCPU_LR(r13)
192 203
193 /* 204 /*
194 * In order for us to easily get the last instruction, 205 * In order for us to easily get the last instruction,
@@ -218,7 +229,7 @@ ld_last_inst:
218 /* Set guest mode to 'jump over instruction' so if lwz faults 229 /* Set guest mode to 'jump over instruction' so if lwz faults
219 * we'll just continue at the next IP. */ 230 * we'll just continue at the next IP. */
220 li r9, KVM_GUEST_MODE_SKIP 231 li r9, KVM_GUEST_MODE_SKIP
221 stb r9, (SHADOW_VCPU_OFF + SVCPU_IN_GUEST)(r13) 232 stb r9, HSTATE_IN_GUEST(r13)
222 233
223 /* 1) enable paging for data */ 234 /* 1) enable paging for data */
224 mfmsr r9 235 mfmsr r9
@@ -232,13 +243,13 @@ ld_last_inst:
232 sync 243 sync
233 244
234#endif 245#endif
235 stw r0, (SHADOW_VCPU_OFF + SVCPU_LAST_INST)(r13) 246 stw r0, SVCPU_LAST_INST(r13)
236 247
237no_ld_last_inst: 248no_ld_last_inst:
238 249
239 /* Unset guest mode */ 250 /* Unset guest mode */
240 li r9, KVM_GUEST_MODE_NONE 251 li r9, KVM_GUEST_MODE_NONE
241 stb r9, (SHADOW_VCPU_OFF + SVCPU_IN_GUEST)(r13) 252 stb r9, HSTATE_IN_GUEST(r13)
242 253
243 /* Switch back to host MMU */ 254 /* Switch back to host MMU */
244 LOAD_HOST_SEGMENTS 255 LOAD_HOST_SEGMENTS
@@ -248,7 +259,7 @@ no_ld_last_inst:
248 * R1 = host R1 259 * R1 = host R1
249 * R2 = host R2 260 * R2 = host R2
250 * R12 = exit handler id 261 * R12 = exit handler id
251 * R13 = shadow vcpu - SHADOW_VCPU_OFF [=PACA on PPC64] 262 * R13 = shadow vcpu (32-bit) or PACA (64-bit)
252 * SVCPU.* = guest * 263 * SVCPU.* = guest *
253 * 264 *
254 */ 265 */
@@ -258,7 +269,7 @@ no_ld_last_inst:
258 ori r7, r7, MSR_IR|MSR_DR|MSR_RI|MSR_ME /* Enable paging */ 269 ori r7, r7, MSR_IR|MSR_DR|MSR_RI|MSR_ME /* Enable paging */
259 mtsrr1 r7 270 mtsrr1 r7
260 /* Load highmem handler address */ 271 /* Load highmem handler address */
261 PPC_LL r8, (SHADOW_VCPU_OFF + SVCPU_VMHANDLER)(r13) 272 PPC_LL r8, HSTATE_VMHANDLER(r13)
262 mtsrr0 r8 273 mtsrr0 r8
263 274
264 RFI 275 RFI
diff --git a/arch/powerpc/kvm/booke.c b/arch/powerpc/kvm/booke.c
index 8462b3a1c1c7..ee45fa01220e 100644
--- a/arch/powerpc/kvm/booke.c
+++ b/arch/powerpc/kvm/booke.c
@@ -13,6 +13,7 @@
13 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 13 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
14 * 14 *
15 * Copyright IBM Corp. 2007 15 * Copyright IBM Corp. 2007
16 * Copyright 2010-2011 Freescale Semiconductor, Inc.
16 * 17 *
17 * Authors: Hollis Blanchard <hollisb@us.ibm.com> 18 * Authors: Hollis Blanchard <hollisb@us.ibm.com>
18 * Christian Ehrhardt <ehrhardt@linux.vnet.ibm.com> 19 * Christian Ehrhardt <ehrhardt@linux.vnet.ibm.com>
@@ -78,6 +79,60 @@ void kvmppc_dump_vcpu(struct kvm_vcpu *vcpu)
78 } 79 }
79} 80}
80 81
82#ifdef CONFIG_SPE
83void kvmppc_vcpu_disable_spe(struct kvm_vcpu *vcpu)
84{
85 preempt_disable();
86 enable_kernel_spe();
87 kvmppc_save_guest_spe(vcpu);
88 vcpu->arch.shadow_msr &= ~MSR_SPE;
89 preempt_enable();
90}
91
92static void kvmppc_vcpu_enable_spe(struct kvm_vcpu *vcpu)
93{
94 preempt_disable();
95 enable_kernel_spe();
96 kvmppc_load_guest_spe(vcpu);
97 vcpu->arch.shadow_msr |= MSR_SPE;
98 preempt_enable();
99}
100
101static void kvmppc_vcpu_sync_spe(struct kvm_vcpu *vcpu)
102{
103 if (vcpu->arch.shared->msr & MSR_SPE) {
104 if (!(vcpu->arch.shadow_msr & MSR_SPE))
105 kvmppc_vcpu_enable_spe(vcpu);
106 } else if (vcpu->arch.shadow_msr & MSR_SPE) {
107 kvmppc_vcpu_disable_spe(vcpu);
108 }
109}
110#else
111static void kvmppc_vcpu_sync_spe(struct kvm_vcpu *vcpu)
112{
113}
114#endif
115
116/*
117 * Helper function for "full" MSR writes. No need to call this if only
118 * EE/CE/ME/DE/RI are changing.
119 */
120void kvmppc_set_msr(struct kvm_vcpu *vcpu, u32 new_msr)
121{
122 u32 old_msr = vcpu->arch.shared->msr;
123
124 vcpu->arch.shared->msr = new_msr;
125
126 kvmppc_mmu_msr_notify(vcpu, old_msr);
127
128 if (vcpu->arch.shared->msr & MSR_WE) {
129 kvm_vcpu_block(vcpu);
130 kvmppc_set_exit_type(vcpu, EMULATED_MTMSRWE_EXITS);
131 };
132
133 kvmppc_vcpu_sync_spe(vcpu);
134}
135
81static void kvmppc_booke_queue_irqprio(struct kvm_vcpu *vcpu, 136static void kvmppc_booke_queue_irqprio(struct kvm_vcpu *vcpu,
82 unsigned int priority) 137 unsigned int priority)
83{ 138{
@@ -257,6 +312,19 @@ void kvmppc_core_deliver_interrupts(struct kvm_vcpu *vcpu)
257 vcpu->arch.shared->int_pending = 0; 312 vcpu->arch.shared->int_pending = 0;
258} 313}
259 314
315int kvmppc_vcpu_run(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
316{
317 int ret;
318
319 local_irq_disable();
320 kvm_guest_enter();
321 ret = __kvmppc_vcpu_run(kvm_run, vcpu);
322 kvm_guest_exit();
323 local_irq_enable();
324
325 return ret;
326}
327
260/** 328/**
261 * kvmppc_handle_exit 329 * kvmppc_handle_exit
262 * 330 *
@@ -344,10 +412,16 @@ int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu,
344 r = RESUME_GUEST; 412 r = RESUME_GUEST;
345 break; 413 break;
346 414
347 case BOOKE_INTERRUPT_SPE_UNAVAIL: 415#ifdef CONFIG_SPE
348 kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_SPE_UNAVAIL); 416 case BOOKE_INTERRUPT_SPE_UNAVAIL: {
417 if (vcpu->arch.shared->msr & MSR_SPE)
418 kvmppc_vcpu_enable_spe(vcpu);
419 else
420 kvmppc_booke_queue_irqprio(vcpu,
421 BOOKE_IRQPRIO_SPE_UNAVAIL);
349 r = RESUME_GUEST; 422 r = RESUME_GUEST;
350 break; 423 break;
424 }
351 425
352 case BOOKE_INTERRUPT_SPE_FP_DATA: 426 case BOOKE_INTERRUPT_SPE_FP_DATA:
353 kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_SPE_FP_DATA); 427 kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_SPE_FP_DATA);
@@ -358,6 +432,28 @@ int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu,
358 kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_SPE_FP_ROUND); 432 kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_SPE_FP_ROUND);
359 r = RESUME_GUEST; 433 r = RESUME_GUEST;
360 break; 434 break;
435#else
436 case BOOKE_INTERRUPT_SPE_UNAVAIL:
437 /*
438 * Guest wants SPE, but host kernel doesn't support it. Send
439 * an "unimplemented operation" program check to the guest.
440 */
441 kvmppc_core_queue_program(vcpu, ESR_PUO | ESR_SPV);
442 r = RESUME_GUEST;
443 break;
444
445 /*
446 * These really should never happen without CONFIG_SPE,
447 * as we should never enable the real MSR[SPE] in the guest.
448 */
449 case BOOKE_INTERRUPT_SPE_FP_DATA:
450 case BOOKE_INTERRUPT_SPE_FP_ROUND:
451 printk(KERN_CRIT "%s: unexpected SPE interrupt %u at %08lx\n",
452 __func__, exit_nr, vcpu->arch.pc);
453 run->hw.hardware_exit_reason = exit_nr;
454 r = RESUME_HOST;
455 break;
456#endif
361 457
362 case BOOKE_INTERRUPT_DATA_STORAGE: 458 case BOOKE_INTERRUPT_DATA_STORAGE:
363 kvmppc_core_queue_data_storage(vcpu, vcpu->arch.fault_dear, 459 kvmppc_core_queue_data_storage(vcpu, vcpu->arch.fault_dear,
@@ -392,6 +488,17 @@ int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu,
392 gpa_t gpaddr; 488 gpa_t gpaddr;
393 gfn_t gfn; 489 gfn_t gfn;
394 490
491#ifdef CONFIG_KVM_E500
492 if (!(vcpu->arch.shared->msr & MSR_PR) &&
493 (eaddr & PAGE_MASK) == vcpu->arch.magic_page_ea) {
494 kvmppc_map_magic(vcpu);
495 kvmppc_account_exit(vcpu, DTLB_VIRT_MISS_EXITS);
496 r = RESUME_GUEST;
497
498 break;
499 }
500#endif
501
395 /* Check the guest TLB. */ 502 /* Check the guest TLB. */
396 gtlb_index = kvmppc_mmu_dtlb_index(vcpu, eaddr); 503 gtlb_index = kvmppc_mmu_dtlb_index(vcpu, eaddr);
397 if (gtlb_index < 0) { 504 if (gtlb_index < 0) {
@@ -514,6 +621,7 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
514 621
515 vcpu->arch.pc = 0; 622 vcpu->arch.pc = 0;
516 vcpu->arch.shared->msr = 0; 623 vcpu->arch.shared->msr = 0;
624 vcpu->arch.shadow_msr = MSR_USER | MSR_DE | MSR_IS | MSR_DS;
517 kvmppc_set_gpr(vcpu, 1, (16<<20) - 8); /* -8 for the callee-save LR slot */ 625 kvmppc_set_gpr(vcpu, 1, (16<<20) - 8); /* -8 for the callee-save LR slot */
518 626
519 vcpu->arch.shadow_pid = 1; 627 vcpu->arch.shadow_pid = 1;
@@ -770,6 +878,26 @@ int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
770 return -ENOTSUPP; 878 return -ENOTSUPP;
771} 879}
772 880
881int kvmppc_core_prepare_memory_region(struct kvm *kvm,
882 struct kvm_userspace_memory_region *mem)
883{
884 return 0;
885}
886
887void kvmppc_core_commit_memory_region(struct kvm *kvm,
888 struct kvm_userspace_memory_region *mem)
889{
890}
891
892int kvmppc_core_init_vm(struct kvm *kvm)
893{
894 return 0;
895}
896
897void kvmppc_core_destroy_vm(struct kvm *kvm)
898{
899}
900
773int __init kvmppc_booke_init(void) 901int __init kvmppc_booke_init(void)
774{ 902{
775 unsigned long ivor[16]; 903 unsigned long ivor[16];
diff --git a/arch/powerpc/kvm/booke.h b/arch/powerpc/kvm/booke.h
index 492bb7030358..8e1fe33d64e5 100644
--- a/arch/powerpc/kvm/booke.h
+++ b/arch/powerpc/kvm/booke.h
@@ -52,24 +52,19 @@
52 52
53extern unsigned long kvmppc_booke_handlers; 53extern unsigned long kvmppc_booke_handlers;
54 54
55/* Helper function for "full" MSR writes. No need to call this if only EE is 55void kvmppc_set_msr(struct kvm_vcpu *vcpu, u32 new_msr);
56 * changing. */ 56void kvmppc_mmu_msr_notify(struct kvm_vcpu *vcpu, u32 old_msr);
57static inline void kvmppc_set_msr(struct kvm_vcpu *vcpu, u32 new_msr)
58{
59 if ((new_msr & MSR_PR) != (vcpu->arch.shared->msr & MSR_PR))
60 kvmppc_mmu_priv_switch(vcpu, new_msr & MSR_PR);
61
62 vcpu->arch.shared->msr = new_msr;
63
64 if (vcpu->arch.shared->msr & MSR_WE) {
65 kvm_vcpu_block(vcpu);
66 kvmppc_set_exit_type(vcpu, EMULATED_MTMSRWE_EXITS);
67 };
68}
69 57
70int kvmppc_booke_emulate_op(struct kvm_run *run, struct kvm_vcpu *vcpu, 58int kvmppc_booke_emulate_op(struct kvm_run *run, struct kvm_vcpu *vcpu,
71 unsigned int inst, int *advance); 59 unsigned int inst, int *advance);
72int kvmppc_booke_emulate_mfspr(struct kvm_vcpu *vcpu, int sprn, int rt); 60int kvmppc_booke_emulate_mfspr(struct kvm_vcpu *vcpu, int sprn, int rt);
73int kvmppc_booke_emulate_mtspr(struct kvm_vcpu *vcpu, int sprn, int rs); 61int kvmppc_booke_emulate_mtspr(struct kvm_vcpu *vcpu, int sprn, int rs);
74 62
63/* low-level asm code to transfer guest state */
64void kvmppc_load_guest_spe(struct kvm_vcpu *vcpu);
65void kvmppc_save_guest_spe(struct kvm_vcpu *vcpu);
66
67/* high-level function, manages flags, host state */
68void kvmppc_vcpu_disable_spe(struct kvm_vcpu *vcpu);
69
75#endif /* __KVM_BOOKE_H__ */ 70#endif /* __KVM_BOOKE_H__ */
diff --git a/arch/powerpc/kvm/booke_interrupts.S b/arch/powerpc/kvm/booke_interrupts.S
index b58ccae95904..42f2fb1f66e9 100644
--- a/arch/powerpc/kvm/booke_interrupts.S
+++ b/arch/powerpc/kvm/booke_interrupts.S
@@ -13,6 +13,7 @@
13 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 13 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
14 * 14 *
15 * Copyright IBM Corp. 2007 15 * Copyright IBM Corp. 2007
16 * Copyright 2011 Freescale Semiconductor, Inc.
16 * 17 *
17 * Authors: Hollis Blanchard <hollisb@us.ibm.com> 18 * Authors: Hollis Blanchard <hollisb@us.ibm.com>
18 */ 19 */
@@ -24,8 +25,6 @@
24#include <asm/page.h> 25#include <asm/page.h>
25#include <asm/asm-offsets.h> 26#include <asm/asm-offsets.h>
26 27
27#define KVMPPC_MSR_MASK (MSR_CE|MSR_EE|MSR_PR|MSR_DE|MSR_ME|MSR_IS|MSR_DS)
28
29#define VCPU_GPR(n) (VCPU_GPRS + (n * 4)) 28#define VCPU_GPR(n) (VCPU_GPRS + (n * 4))
30 29
31/* The host stack layout: */ 30/* The host stack layout: */
@@ -192,6 +191,12 @@ _GLOBAL(kvmppc_resume_host)
192 lwz r3, VCPU_HOST_PID(r4) 191 lwz r3, VCPU_HOST_PID(r4)
193 mtspr SPRN_PID, r3 192 mtspr SPRN_PID, r3
194 193
194#ifdef CONFIG_FSL_BOOKE
195 /* we cheat and know that Linux doesn't use PID1 which is always 0 */
196 lis r3, 0
197 mtspr SPRN_PID1, r3
198#endif
199
195 /* Restore host IVPR before re-enabling interrupts. We cheat and know 200 /* Restore host IVPR before re-enabling interrupts. We cheat and know
196 * that Linux IVPR is always 0xc0000000. */ 201 * that Linux IVPR is always 0xc0000000. */
197 lis r3, 0xc000 202 lis r3, 0xc000
@@ -241,6 +246,14 @@ _GLOBAL(kvmppc_resume_host)
241heavyweight_exit: 246heavyweight_exit:
242 /* Not returning to guest. */ 247 /* Not returning to guest. */
243 248
249#ifdef CONFIG_SPE
250 /* save guest SPEFSCR and load host SPEFSCR */
251 mfspr r9, SPRN_SPEFSCR
252 stw r9, VCPU_SPEFSCR(r4)
253 lwz r9, VCPU_HOST_SPEFSCR(r4)
254 mtspr SPRN_SPEFSCR, r9
255#endif
256
244 /* We already saved guest volatile register state; now save the 257 /* We already saved guest volatile register state; now save the
245 * non-volatiles. */ 258 * non-volatiles. */
246 stw r15, VCPU_GPR(r15)(r4) 259 stw r15, VCPU_GPR(r15)(r4)
@@ -342,6 +355,14 @@ _GLOBAL(__kvmppc_vcpu_run)
342 lwz r30, VCPU_GPR(r30)(r4) 355 lwz r30, VCPU_GPR(r30)(r4)
343 lwz r31, VCPU_GPR(r31)(r4) 356 lwz r31, VCPU_GPR(r31)(r4)
344 357
358#ifdef CONFIG_SPE
359 /* save host SPEFSCR and load guest SPEFSCR */
360 mfspr r3, SPRN_SPEFSCR
361 stw r3, VCPU_HOST_SPEFSCR(r4)
362 lwz r3, VCPU_SPEFSCR(r4)
363 mtspr SPRN_SPEFSCR, r3
364#endif
365
345lightweight_exit: 366lightweight_exit:
346 stw r2, HOST_R2(r1) 367 stw r2, HOST_R2(r1)
347 368
@@ -350,6 +371,11 @@ lightweight_exit:
350 lwz r3, VCPU_SHADOW_PID(r4) 371 lwz r3, VCPU_SHADOW_PID(r4)
351 mtspr SPRN_PID, r3 372 mtspr SPRN_PID, r3
352 373
374#ifdef CONFIG_FSL_BOOKE
375 lwz r3, VCPU_SHADOW_PID1(r4)
376 mtspr SPRN_PID1, r3
377#endif
378
353#ifdef CONFIG_44x 379#ifdef CONFIG_44x
354 iccci 0, 0 /* XXX hack */ 380 iccci 0, 0 /* XXX hack */
355#endif 381#endif
@@ -405,20 +431,17 @@ lightweight_exit:
405 431
406 /* Finish loading guest volatiles and jump to guest. */ 432 /* Finish loading guest volatiles and jump to guest. */
407 lwz r3, VCPU_CTR(r4) 433 lwz r3, VCPU_CTR(r4)
434 lwz r5, VCPU_CR(r4)
435 lwz r6, VCPU_PC(r4)
436 lwz r7, VCPU_SHADOW_MSR(r4)
408 mtctr r3 437 mtctr r3
409 lwz r3, VCPU_CR(r4) 438 mtcr r5
410 mtcr r3 439 mtsrr0 r6
440 mtsrr1 r7
411 lwz r5, VCPU_GPR(r5)(r4) 441 lwz r5, VCPU_GPR(r5)(r4)
412 lwz r6, VCPU_GPR(r6)(r4) 442 lwz r6, VCPU_GPR(r6)(r4)
413 lwz r7, VCPU_GPR(r7)(r4) 443 lwz r7, VCPU_GPR(r7)(r4)
414 lwz r8, VCPU_GPR(r8)(r4) 444 lwz r8, VCPU_GPR(r8)(r4)
415 lwz r3, VCPU_PC(r4)
416 mtsrr0 r3
417 lwz r3, VCPU_SHARED(r4)
418 lwz r3, (VCPU_SHARED_MSR + 4)(r3)
419 oris r3, r3, KVMPPC_MSR_MASK@h
420 ori r3, r3, KVMPPC_MSR_MASK@l
421 mtsrr1 r3
422 445
423 /* Clear any debug events which occurred since we disabled MSR[DE]. 446 /* Clear any debug events which occurred since we disabled MSR[DE].
424 * XXX This gives us a 3-instruction window in which a breakpoint 447 * XXX This gives us a 3-instruction window in which a breakpoint
@@ -430,3 +453,24 @@ lightweight_exit:
430 lwz r3, VCPU_GPR(r3)(r4) 453 lwz r3, VCPU_GPR(r3)(r4)
431 lwz r4, VCPU_GPR(r4)(r4) 454 lwz r4, VCPU_GPR(r4)(r4)
432 rfi 455 rfi
456
457#ifdef CONFIG_SPE
458_GLOBAL(kvmppc_save_guest_spe)
459 cmpi 0,r3,0
460 beqlr-
461 SAVE_32EVRS(0, r4, r3, VCPU_EVR)
462 evxor evr6, evr6, evr6
463 evmwumiaa evr6, evr6, evr6
464 li r4,VCPU_ACC
465 evstddx evr6, r4, r3 /* save acc */
466 blr
467
468_GLOBAL(kvmppc_load_guest_spe)
469 cmpi 0,r3,0
470 beqlr-
471 li r4,VCPU_ACC
472 evlddx evr6,r4,r3
473 evmra evr6,evr6 /* load acc */
474 REST_32EVRS(0, r4, r3, VCPU_EVR)
475 blr
476#endif
diff --git a/arch/powerpc/kvm/e500.c b/arch/powerpc/kvm/e500.c
index 318dbc61ba44..797a7447c268 100644
--- a/arch/powerpc/kvm/e500.c
+++ b/arch/powerpc/kvm/e500.c
@@ -1,5 +1,5 @@
1/* 1/*
2 * Copyright (C) 2008 Freescale Semiconductor, Inc. All rights reserved. 2 * Copyright (C) 2008-2011 Freescale Semiconductor, Inc. All rights reserved.
3 * 3 *
4 * Author: Yu Liu, <yu.liu@freescale.com> 4 * Author: Yu Liu, <yu.liu@freescale.com>
5 * 5 *
@@ -41,6 +41,11 @@ void kvmppc_core_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
41void kvmppc_core_vcpu_put(struct kvm_vcpu *vcpu) 41void kvmppc_core_vcpu_put(struct kvm_vcpu *vcpu)
42{ 42{
43 kvmppc_e500_tlb_put(vcpu); 43 kvmppc_e500_tlb_put(vcpu);
44
45#ifdef CONFIG_SPE
46 if (vcpu->arch.shadow_msr & MSR_SPE)
47 kvmppc_vcpu_disable_spe(vcpu);
48#endif
44} 49}
45 50
46int kvmppc_core_check_processor_compat(void) 51int kvmppc_core_check_processor_compat(void)
diff --git a/arch/powerpc/kvm/e500_emulate.c b/arch/powerpc/kvm/e500_emulate.c
index 69cd665a0caf..d48ae396f41e 100644
--- a/arch/powerpc/kvm/e500_emulate.c
+++ b/arch/powerpc/kvm/e500_emulate.c
@@ -81,8 +81,12 @@ int kvmppc_core_emulate_mtspr(struct kvm_vcpu *vcpu, int sprn, int rs)
81 kvmppc_set_pid(vcpu, spr_val); 81 kvmppc_set_pid(vcpu, spr_val);
82 break; 82 break;
83 case SPRN_PID1: 83 case SPRN_PID1:
84 if (spr_val != 0)
85 return EMULATE_FAIL;
84 vcpu_e500->pid[1] = spr_val; break; 86 vcpu_e500->pid[1] = spr_val; break;
85 case SPRN_PID2: 87 case SPRN_PID2:
88 if (spr_val != 0)
89 return EMULATE_FAIL;
86 vcpu_e500->pid[2] = spr_val; break; 90 vcpu_e500->pid[2] = spr_val; break;
87 case SPRN_MAS0: 91 case SPRN_MAS0:
88 vcpu_e500->mas0 = spr_val; break; 92 vcpu_e500->mas0 = spr_val; break;
diff --git a/arch/powerpc/kvm/e500_tlb.c b/arch/powerpc/kvm/e500_tlb.c
index b18fe353397d..13c432ea2fa8 100644
--- a/arch/powerpc/kvm/e500_tlb.c
+++ b/arch/powerpc/kvm/e500_tlb.c
@@ -28,8 +28,196 @@
28 28
29#define to_htlb1_esel(esel) (tlb1_entry_num - (esel) - 1) 29#define to_htlb1_esel(esel) (tlb1_entry_num - (esel) - 1)
30 30
31struct id {
32 unsigned long val;
33 struct id **pentry;
34};
35
36#define NUM_TIDS 256
37
38/*
39 * This table provide mappings from:
40 * (guestAS,guestTID,guestPR) --> ID of physical cpu
41 * guestAS [0..1]
42 * guestTID [0..255]
43 * guestPR [0..1]
44 * ID [1..255]
45 * Each vcpu keeps one vcpu_id_table.
46 */
47struct vcpu_id_table {
48 struct id id[2][NUM_TIDS][2];
49};
50
51/*
52 * This table provide reversed mappings of vcpu_id_table:
53 * ID --> address of vcpu_id_table item.
54 * Each physical core has one pcpu_id_table.
55 */
56struct pcpu_id_table {
57 struct id *entry[NUM_TIDS];
58};
59
60static DEFINE_PER_CPU(struct pcpu_id_table, pcpu_sids);
61
62/* This variable keeps last used shadow ID on local core.
63 * The valid range of shadow ID is [1..255] */
64static DEFINE_PER_CPU(unsigned long, pcpu_last_used_sid);
65
31static unsigned int tlb1_entry_num; 66static unsigned int tlb1_entry_num;
32 67
68/*
69 * Allocate a free shadow id and setup a valid sid mapping in given entry.
70 * A mapping is only valid when vcpu_id_table and pcpu_id_table are match.
71 *
72 * The caller must have preemption disabled, and keep it that way until
73 * it has finished with the returned shadow id (either written into the
74 * TLB or arch.shadow_pid, or discarded).
75 */
76static inline int local_sid_setup_one(struct id *entry)
77{
78 unsigned long sid;
79 int ret = -1;
80
81 sid = ++(__get_cpu_var(pcpu_last_used_sid));
82 if (sid < NUM_TIDS) {
83 __get_cpu_var(pcpu_sids).entry[sid] = entry;
84 entry->val = sid;
85 entry->pentry = &__get_cpu_var(pcpu_sids).entry[sid];
86 ret = sid;
87 }
88
89 /*
90 * If sid == NUM_TIDS, we've run out of sids. We return -1, and
91 * the caller will invalidate everything and start over.
92 *
93 * sid > NUM_TIDS indicates a race, which we disable preemption to
94 * avoid.
95 */
96 WARN_ON(sid > NUM_TIDS);
97
98 return ret;
99}
100
101/*
102 * Check if given entry contain a valid shadow id mapping.
103 * An ID mapping is considered valid only if
104 * both vcpu and pcpu know this mapping.
105 *
106 * The caller must have preemption disabled, and keep it that way until
107 * it has finished with the returned shadow id (either written into the
108 * TLB or arch.shadow_pid, or discarded).
109 */
110static inline int local_sid_lookup(struct id *entry)
111{
112 if (entry && entry->val != 0 &&
113 __get_cpu_var(pcpu_sids).entry[entry->val] == entry &&
114 entry->pentry == &__get_cpu_var(pcpu_sids).entry[entry->val])
115 return entry->val;
116 return -1;
117}
118
119/* Invalidate all id mappings on local core */
120static inline void local_sid_destroy_all(void)
121{
122 preempt_disable();
123 __get_cpu_var(pcpu_last_used_sid) = 0;
124 memset(&__get_cpu_var(pcpu_sids), 0, sizeof(__get_cpu_var(pcpu_sids)));
125 preempt_enable();
126}
127
128static void *kvmppc_e500_id_table_alloc(struct kvmppc_vcpu_e500 *vcpu_e500)
129{
130 vcpu_e500->idt = kzalloc(sizeof(struct vcpu_id_table), GFP_KERNEL);
131 return vcpu_e500->idt;
132}
133
134static void kvmppc_e500_id_table_free(struct kvmppc_vcpu_e500 *vcpu_e500)
135{
136 kfree(vcpu_e500->idt);
137}
138
139/* Invalidate all mappings on vcpu */
140static void kvmppc_e500_id_table_reset_all(struct kvmppc_vcpu_e500 *vcpu_e500)
141{
142 memset(vcpu_e500->idt, 0, sizeof(struct vcpu_id_table));
143
144 /* Update shadow pid when mappings are changed */
145 kvmppc_e500_recalc_shadow_pid(vcpu_e500);
146}
147
148/* Invalidate one ID mapping on vcpu */
149static inline void kvmppc_e500_id_table_reset_one(
150 struct kvmppc_vcpu_e500 *vcpu_e500,
151 int as, int pid, int pr)
152{
153 struct vcpu_id_table *idt = vcpu_e500->idt;
154
155 BUG_ON(as >= 2);
156 BUG_ON(pid >= NUM_TIDS);
157 BUG_ON(pr >= 2);
158
159 idt->id[as][pid][pr].val = 0;
160 idt->id[as][pid][pr].pentry = NULL;
161
162 /* Update shadow pid when mappings are changed */
163 kvmppc_e500_recalc_shadow_pid(vcpu_e500);
164}
165
166/*
167 * Map guest (vcpu,AS,ID,PR) to physical core shadow id.
168 * This function first lookup if a valid mapping exists,
169 * if not, then creates a new one.
170 *
171 * The caller must have preemption disabled, and keep it that way until
172 * it has finished with the returned shadow id (either written into the
173 * TLB or arch.shadow_pid, or discarded).
174 */
175static unsigned int kvmppc_e500_get_sid(struct kvmppc_vcpu_e500 *vcpu_e500,
176 unsigned int as, unsigned int gid,
177 unsigned int pr, int avoid_recursion)
178{
179 struct vcpu_id_table *idt = vcpu_e500->idt;
180 int sid;
181
182 BUG_ON(as >= 2);
183 BUG_ON(gid >= NUM_TIDS);
184 BUG_ON(pr >= 2);
185
186 sid = local_sid_lookup(&idt->id[as][gid][pr]);
187
188 while (sid <= 0) {
189 /* No mapping yet */
190 sid = local_sid_setup_one(&idt->id[as][gid][pr]);
191 if (sid <= 0) {
192 _tlbil_all();
193 local_sid_destroy_all();
194 }
195
196 /* Update shadow pid when mappings are changed */
197 if (!avoid_recursion)
198 kvmppc_e500_recalc_shadow_pid(vcpu_e500);
199 }
200
201 return sid;
202}
203
204/* Map guest pid to shadow.
205 * We use PID to keep shadow of current guest non-zero PID,
206 * and use PID1 to keep shadow of guest zero PID.
207 * So that guest tlbe with TID=0 can be accessed at any time */
208void kvmppc_e500_recalc_shadow_pid(struct kvmppc_vcpu_e500 *vcpu_e500)
209{
210 preempt_disable();
211 vcpu_e500->vcpu.arch.shadow_pid = kvmppc_e500_get_sid(vcpu_e500,
212 get_cur_as(&vcpu_e500->vcpu),
213 get_cur_pid(&vcpu_e500->vcpu),
214 get_cur_pr(&vcpu_e500->vcpu), 1);
215 vcpu_e500->vcpu.arch.shadow_pid1 = kvmppc_e500_get_sid(vcpu_e500,
216 get_cur_as(&vcpu_e500->vcpu), 0,
217 get_cur_pr(&vcpu_e500->vcpu), 1);
218 preempt_enable();
219}
220
33void kvmppc_dump_tlbs(struct kvm_vcpu *vcpu) 221void kvmppc_dump_tlbs(struct kvm_vcpu *vcpu)
34{ 222{
35 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 223 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
@@ -41,25 +229,14 @@ void kvmppc_dump_tlbs(struct kvm_vcpu *vcpu)
41 229
42 for (tlbsel = 0; tlbsel < 2; tlbsel++) { 230 for (tlbsel = 0; tlbsel < 2; tlbsel++) {
43 printk("Guest TLB%d:\n", tlbsel); 231 printk("Guest TLB%d:\n", tlbsel);
44 for (i = 0; i < vcpu_e500->guest_tlb_size[tlbsel]; i++) { 232 for (i = 0; i < vcpu_e500->gtlb_size[tlbsel]; i++) {
45 tlbe = &vcpu_e500->guest_tlb[tlbsel][i]; 233 tlbe = &vcpu_e500->gtlb_arch[tlbsel][i];
46 if (tlbe->mas1 & MAS1_VALID) 234 if (tlbe->mas1 & MAS1_VALID)
47 printk(" G[%d][%3d] | %08X | %08X | %08X | %08X |\n", 235 printk(" G[%d][%3d] | %08X | %08X | %08X | %08X |\n",
48 tlbsel, i, tlbe->mas1, tlbe->mas2, 236 tlbsel, i, tlbe->mas1, tlbe->mas2,
49 tlbe->mas3, tlbe->mas7); 237 tlbe->mas3, tlbe->mas7);
50 } 238 }
51 } 239 }
52
53 for (tlbsel = 0; tlbsel < 2; tlbsel++) {
54 printk("Shadow TLB%d:\n", tlbsel);
55 for (i = 0; i < vcpu_e500->shadow_tlb_size[tlbsel]; i++) {
56 tlbe = &vcpu_e500->shadow_tlb[tlbsel][i];
57 if (tlbe->mas1 & MAS1_VALID)
58 printk(" S[%d][%3d] | %08X | %08X | %08X | %08X |\n",
59 tlbsel, i, tlbe->mas1, tlbe->mas2,
60 tlbe->mas3, tlbe->mas7);
61 }
62 }
63} 240}
64 241
65static inline unsigned int tlb0_get_next_victim( 242static inline unsigned int tlb0_get_next_victim(
@@ -67,16 +244,17 @@ static inline unsigned int tlb0_get_next_victim(
67{ 244{
68 unsigned int victim; 245 unsigned int victim;
69 246
70 victim = vcpu_e500->guest_tlb_nv[0]++; 247 victim = vcpu_e500->gtlb_nv[0]++;
71 if (unlikely(vcpu_e500->guest_tlb_nv[0] >= KVM_E500_TLB0_WAY_NUM)) 248 if (unlikely(vcpu_e500->gtlb_nv[0] >= KVM_E500_TLB0_WAY_NUM))
72 vcpu_e500->guest_tlb_nv[0] = 0; 249 vcpu_e500->gtlb_nv[0] = 0;
73 250
74 return victim; 251 return victim;
75} 252}
76 253
77static inline unsigned int tlb1_max_shadow_size(void) 254static inline unsigned int tlb1_max_shadow_size(void)
78{ 255{
79 return tlb1_entry_num - tlbcam_index; 256 /* reserve one entry for magic page */
257 return tlb1_entry_num - tlbcam_index - 1;
80} 258}
81 259
82static inline int tlbe_is_writable(struct tlbe *tlbe) 260static inline int tlbe_is_writable(struct tlbe *tlbe)
@@ -112,72 +290,149 @@ static inline u32 e500_shadow_mas2_attrib(u32 mas2, int usermode)
112/* 290/*
113 * writing shadow tlb entry to host TLB 291 * writing shadow tlb entry to host TLB
114 */ 292 */
115static inline void __write_host_tlbe(struct tlbe *stlbe) 293static inline void __write_host_tlbe(struct tlbe *stlbe, uint32_t mas0)
116{ 294{
295 unsigned long flags;
296
297 local_irq_save(flags);
298 mtspr(SPRN_MAS0, mas0);
117 mtspr(SPRN_MAS1, stlbe->mas1); 299 mtspr(SPRN_MAS1, stlbe->mas1);
118 mtspr(SPRN_MAS2, stlbe->mas2); 300 mtspr(SPRN_MAS2, stlbe->mas2);
119 mtspr(SPRN_MAS3, stlbe->mas3); 301 mtspr(SPRN_MAS3, stlbe->mas3);
120 mtspr(SPRN_MAS7, stlbe->mas7); 302 mtspr(SPRN_MAS7, stlbe->mas7);
121 __asm__ __volatile__ ("tlbwe\n" : : ); 303 asm volatile("isync; tlbwe" : : : "memory");
304 local_irq_restore(flags);
122} 305}
123 306
124static inline void write_host_tlbe(struct kvmppc_vcpu_e500 *vcpu_e500, 307static inline void write_host_tlbe(struct kvmppc_vcpu_e500 *vcpu_e500,
125 int tlbsel, int esel) 308 int tlbsel, int esel, struct tlbe *stlbe)
126{ 309{
127 struct tlbe *stlbe = &vcpu_e500->shadow_tlb[tlbsel][esel];
128
129 local_irq_disable();
130 if (tlbsel == 0) { 310 if (tlbsel == 0) {
131 __write_host_tlbe(stlbe); 311 __write_host_tlbe(stlbe,
312 MAS0_TLBSEL(0) |
313 MAS0_ESEL(esel & (KVM_E500_TLB0_WAY_NUM - 1)));
132 } else { 314 } else {
133 unsigned register mas0; 315 __write_host_tlbe(stlbe,
134 316 MAS0_TLBSEL(1) |
135 mas0 = mfspr(SPRN_MAS0); 317 MAS0_ESEL(to_htlb1_esel(esel)));
136
137 mtspr(SPRN_MAS0, MAS0_TLBSEL(1) | MAS0_ESEL(to_htlb1_esel(esel)));
138 __write_host_tlbe(stlbe);
139
140 mtspr(SPRN_MAS0, mas0);
141 } 318 }
142 local_irq_enable(); 319 trace_kvm_stlb_write(index_of(tlbsel, esel), stlbe->mas1, stlbe->mas2,
320 stlbe->mas3, stlbe->mas7);
321}
322
323void kvmppc_map_magic(struct kvm_vcpu *vcpu)
324{
325 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
326 struct tlbe magic;
327 ulong shared_page = ((ulong)vcpu->arch.shared) & PAGE_MASK;
328 unsigned int stid;
329 pfn_t pfn;
330
331 pfn = (pfn_t)virt_to_phys((void *)shared_page) >> PAGE_SHIFT;
332 get_page(pfn_to_page(pfn));
333
334 preempt_disable();
335 stid = kvmppc_e500_get_sid(vcpu_e500, 0, 0, 0, 0);
336
337 magic.mas1 = MAS1_VALID | MAS1_TS | MAS1_TID(stid) |
338 MAS1_TSIZE(BOOK3E_PAGESZ_4K);
339 magic.mas2 = vcpu->arch.magic_page_ea | MAS2_M;
340 magic.mas3 = (pfn << PAGE_SHIFT) |
341 MAS3_SW | MAS3_SR | MAS3_UW | MAS3_UR;
342 magic.mas7 = pfn >> (32 - PAGE_SHIFT);
343
344 __write_host_tlbe(&magic, MAS0_TLBSEL(1) | MAS0_ESEL(tlbcam_index));
345 preempt_enable();
143} 346}
144 347
145void kvmppc_e500_tlb_load(struct kvm_vcpu *vcpu, int cpu) 348void kvmppc_e500_tlb_load(struct kvm_vcpu *vcpu, int cpu)
146{ 349{
147 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 350 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
148 int i; 351
149 unsigned register mas0; 352 /* Shadow PID may be expired on local core */
150 353 kvmppc_e500_recalc_shadow_pid(vcpu_e500);
151 /* Load all valid TLB1 entries to reduce guest tlb miss fault */
152 local_irq_disable();
153 mas0 = mfspr(SPRN_MAS0);
154 for (i = 0; i < tlb1_max_shadow_size(); i++) {
155 struct tlbe *stlbe = &vcpu_e500->shadow_tlb[1][i];
156
157 if (get_tlb_v(stlbe)) {
158 mtspr(SPRN_MAS0, MAS0_TLBSEL(1)
159 | MAS0_ESEL(to_htlb1_esel(i)));
160 __write_host_tlbe(stlbe);
161 }
162 }
163 mtspr(SPRN_MAS0, mas0);
164 local_irq_enable();
165} 354}
166 355
167void kvmppc_e500_tlb_put(struct kvm_vcpu *vcpu) 356void kvmppc_e500_tlb_put(struct kvm_vcpu *vcpu)
168{ 357{
169 _tlbil_all(); 358}
359
360static void kvmppc_e500_stlbe_invalidate(struct kvmppc_vcpu_e500 *vcpu_e500,
361 int tlbsel, int esel)
362{
363 struct tlbe *gtlbe = &vcpu_e500->gtlb_arch[tlbsel][esel];
364 struct vcpu_id_table *idt = vcpu_e500->idt;
365 unsigned int pr, tid, ts, pid;
366 u32 val, eaddr;
367 unsigned long flags;
368
369 ts = get_tlb_ts(gtlbe);
370 tid = get_tlb_tid(gtlbe);
371
372 preempt_disable();
373
374 /* One guest ID may be mapped to two shadow IDs */
375 for (pr = 0; pr < 2; pr++) {
376 /*
377 * The shadow PID can have a valid mapping on at most one
378 * host CPU. In the common case, it will be valid on this
379 * CPU, in which case (for TLB0) we do a local invalidation
380 * of the specific address.
381 *
382 * If the shadow PID is not valid on the current host CPU, or
383 * if we're invalidating a TLB1 entry, we invalidate the
384 * entire shadow PID.
385 */
386 if (tlbsel == 1 ||
387 (pid = local_sid_lookup(&idt->id[ts][tid][pr])) <= 0) {
388 kvmppc_e500_id_table_reset_one(vcpu_e500, ts, tid, pr);
389 continue;
390 }
391
392 /*
393 * The guest is invalidating a TLB0 entry which is in a PID
394 * that has a valid shadow mapping on this host CPU. We
395 * search host TLB0 to invalidate it's shadow TLB entry,
396 * similar to __tlbil_va except that we need to look in AS1.
397 */
398 val = (pid << MAS6_SPID_SHIFT) | MAS6_SAS;
399 eaddr = get_tlb_eaddr(gtlbe);
400
401 local_irq_save(flags);
402
403 mtspr(SPRN_MAS6, val);
404 asm volatile("tlbsx 0, %[eaddr]" : : [eaddr] "r" (eaddr));
405 val = mfspr(SPRN_MAS1);
406 if (val & MAS1_VALID) {
407 mtspr(SPRN_MAS1, val & ~MAS1_VALID);
408 asm volatile("tlbwe");
409 }
410
411 local_irq_restore(flags);
412 }
413
414 preempt_enable();
170} 415}
171 416
172/* Search the guest TLB for a matching entry. */ 417/* Search the guest TLB for a matching entry. */
173static int kvmppc_e500_tlb_index(struct kvmppc_vcpu_e500 *vcpu_e500, 418static int kvmppc_e500_tlb_index(struct kvmppc_vcpu_e500 *vcpu_e500,
174 gva_t eaddr, int tlbsel, unsigned int pid, int as) 419 gva_t eaddr, int tlbsel, unsigned int pid, int as)
175{ 420{
421 int size = vcpu_e500->gtlb_size[tlbsel];
422 int set_base;
176 int i; 423 int i;
177 424
178 /* XXX Replace loop with fancy data structures. */ 425 if (tlbsel == 0) {
179 for (i = 0; i < vcpu_e500->guest_tlb_size[tlbsel]; i++) { 426 int mask = size / KVM_E500_TLB0_WAY_NUM - 1;
180 struct tlbe *tlbe = &vcpu_e500->guest_tlb[tlbsel][i]; 427 set_base = (eaddr >> PAGE_SHIFT) & mask;
428 set_base *= KVM_E500_TLB0_WAY_NUM;
429 size = KVM_E500_TLB0_WAY_NUM;
430 } else {
431 set_base = 0;
432 }
433
434 for (i = 0; i < size; i++) {
435 struct tlbe *tlbe = &vcpu_e500->gtlb_arch[tlbsel][set_base + i];
181 unsigned int tid; 436 unsigned int tid;
182 437
183 if (eaddr < get_tlb_eaddr(tlbe)) 438 if (eaddr < get_tlb_eaddr(tlbe))
@@ -196,66 +451,32 @@ static int kvmppc_e500_tlb_index(struct kvmppc_vcpu_e500 *vcpu_e500,
196 if (get_tlb_ts(tlbe) != as && as != -1) 451 if (get_tlb_ts(tlbe) != as && as != -1)
197 continue; 452 continue;
198 453
199 return i; 454 return set_base + i;
200 } 455 }
201 456
202 return -1; 457 return -1;
203} 458}
204 459
205static void kvmppc_e500_shadow_release(struct kvmppc_vcpu_e500 *vcpu_e500, 460static inline void kvmppc_e500_priv_setup(struct tlbe_priv *priv,
206 int tlbsel, int esel) 461 struct tlbe *gtlbe,
207{ 462 pfn_t pfn)
208 struct tlbe *stlbe = &vcpu_e500->shadow_tlb[tlbsel][esel];
209 struct page *page = vcpu_e500->shadow_pages[tlbsel][esel];
210
211 if (page) {
212 vcpu_e500->shadow_pages[tlbsel][esel] = NULL;
213
214 if (get_tlb_v(stlbe)) {
215 if (tlbe_is_writable(stlbe))
216 kvm_release_page_dirty(page);
217 else
218 kvm_release_page_clean(page);
219 }
220 }
221}
222
223static void kvmppc_e500_stlbe_invalidate(struct kvmppc_vcpu_e500 *vcpu_e500,
224 int tlbsel, int esel)
225{ 463{
226 struct tlbe *stlbe = &vcpu_e500->shadow_tlb[tlbsel][esel]; 464 priv->pfn = pfn;
465 priv->flags = E500_TLB_VALID;
227 466
228 kvmppc_e500_shadow_release(vcpu_e500, tlbsel, esel); 467 if (tlbe_is_writable(gtlbe))
229 stlbe->mas1 = 0; 468 priv->flags |= E500_TLB_DIRTY;
230 trace_kvm_stlb_inval(index_of(tlbsel, esel));
231} 469}
232 470
233static void kvmppc_e500_tlb1_invalidate(struct kvmppc_vcpu_e500 *vcpu_e500, 471static inline void kvmppc_e500_priv_release(struct tlbe_priv *priv)
234 gva_t eaddr, gva_t eend, u32 tid)
235{ 472{
236 unsigned int pid = tid & 0xff; 473 if (priv->flags & E500_TLB_VALID) {
237 unsigned int i; 474 if (priv->flags & E500_TLB_DIRTY)
238 475 kvm_release_pfn_dirty(priv->pfn);
239 /* XXX Replace loop with fancy data structures. */ 476 else
240 for (i = 0; i < vcpu_e500->guest_tlb_size[1]; i++) { 477 kvm_release_pfn_clean(priv->pfn);
241 struct tlbe *stlbe = &vcpu_e500->shadow_tlb[1][i];
242 unsigned int tid;
243
244 if (!get_tlb_v(stlbe))
245 continue;
246
247 if (eend < get_tlb_eaddr(stlbe))
248 continue;
249 478
250 if (eaddr > get_tlb_end(stlbe)) 479 priv->flags = 0;
251 continue;
252
253 tid = get_tlb_tid(stlbe);
254 if (tid && (tid != pid))
255 continue;
256
257 kvmppc_e500_stlbe_invalidate(vcpu_e500, 1, i);
258 write_host_tlbe(vcpu_e500, 1, i);
259 } 480 }
260} 481}
261 482
@@ -273,7 +494,7 @@ static inline void kvmppc_e500_deliver_tlb_miss(struct kvm_vcpu *vcpu,
273 tsized = (vcpu_e500->mas4 >> 7) & 0x1f; 494 tsized = (vcpu_e500->mas4 >> 7) & 0x1f;
274 495
275 vcpu_e500->mas0 = MAS0_TLBSEL(tlbsel) | MAS0_ESEL(victim) 496 vcpu_e500->mas0 = MAS0_TLBSEL(tlbsel) | MAS0_ESEL(victim)
276 | MAS0_NV(vcpu_e500->guest_tlb_nv[tlbsel]); 497 | MAS0_NV(vcpu_e500->gtlb_nv[tlbsel]);
277 vcpu_e500->mas1 = MAS1_VALID | (as ? MAS1_TS : 0) 498 vcpu_e500->mas1 = MAS1_VALID | (as ? MAS1_TS : 0)
278 | MAS1_TID(vcpu_e500->pid[pidsel]) 499 | MAS1_TID(vcpu_e500->pid[pidsel])
279 | MAS1_TSIZE(tsized); 500 | MAS1_TSIZE(tsized);
@@ -286,56 +507,154 @@ static inline void kvmppc_e500_deliver_tlb_miss(struct kvm_vcpu *vcpu,
286 vcpu_e500->mas7 = 0; 507 vcpu_e500->mas7 = 0;
287} 508}
288 509
289static inline void kvmppc_e500_shadow_map(struct kvmppc_vcpu_e500 *vcpu_e500, 510static inline void kvmppc_e500_setup_stlbe(struct kvmppc_vcpu_e500 *vcpu_e500,
290 u64 gvaddr, gfn_t gfn, struct tlbe *gtlbe, int tlbsel, int esel) 511 struct tlbe *gtlbe, int tsize,
512 struct tlbe_priv *priv,
513 u64 gvaddr, struct tlbe *stlbe)
291{ 514{
292 struct page *new_page; 515 pfn_t pfn = priv->pfn;
293 struct tlbe *stlbe; 516 unsigned int stid;
294 hpa_t hpaddr;
295
296 stlbe = &vcpu_e500->shadow_tlb[tlbsel][esel];
297
298 /* Get reference to new page. */
299 new_page = gfn_to_page(vcpu_e500->vcpu.kvm, gfn);
300 if (is_error_page(new_page)) {
301 printk(KERN_ERR "Couldn't get guest page for gfn %lx!\n",
302 (long)gfn);
303 kvm_release_page_clean(new_page);
304 return;
305 }
306 hpaddr = page_to_phys(new_page);
307
308 /* Drop reference to old page. */
309 kvmppc_e500_shadow_release(vcpu_e500, tlbsel, esel);
310 517
311 vcpu_e500->shadow_pages[tlbsel][esel] = new_page; 518 stid = kvmppc_e500_get_sid(vcpu_e500, get_tlb_ts(gtlbe),
519 get_tlb_tid(gtlbe),
520 get_cur_pr(&vcpu_e500->vcpu), 0);
312 521
313 /* Force TS=1 IPROT=0 TSIZE=4KB for all guest mappings. */ 522 /* Force TS=1 IPROT=0 for all guest mappings. */
314 stlbe->mas1 = MAS1_TSIZE(BOOK3E_PAGESZ_4K) 523 stlbe->mas1 = MAS1_TSIZE(tsize)
315 | MAS1_TID(get_tlb_tid(gtlbe)) | MAS1_TS | MAS1_VALID; 524 | MAS1_TID(stid) | MAS1_TS | MAS1_VALID;
316 stlbe->mas2 = (gvaddr & MAS2_EPN) 525 stlbe->mas2 = (gvaddr & MAS2_EPN)
317 | e500_shadow_mas2_attrib(gtlbe->mas2, 526 | e500_shadow_mas2_attrib(gtlbe->mas2,
318 vcpu_e500->vcpu.arch.shared->msr & MSR_PR); 527 vcpu_e500->vcpu.arch.shared->msr & MSR_PR);
319 stlbe->mas3 = (hpaddr & MAS3_RPN) 528 stlbe->mas3 = ((pfn << PAGE_SHIFT) & MAS3_RPN)
320 | e500_shadow_mas3_attrib(gtlbe->mas3, 529 | e500_shadow_mas3_attrib(gtlbe->mas3,
321 vcpu_e500->vcpu.arch.shared->msr & MSR_PR); 530 vcpu_e500->vcpu.arch.shared->msr & MSR_PR);
322 stlbe->mas7 = (hpaddr >> 32) & MAS7_RPN; 531 stlbe->mas7 = (pfn >> (32 - PAGE_SHIFT)) & MAS7_RPN;
532}
323 533
324 trace_kvm_stlb_write(index_of(tlbsel, esel), stlbe->mas1, stlbe->mas2, 534
325 stlbe->mas3, stlbe->mas7); 535static inline void kvmppc_e500_shadow_map(struct kvmppc_vcpu_e500 *vcpu_e500,
536 u64 gvaddr, gfn_t gfn, struct tlbe *gtlbe, int tlbsel, int esel,
537 struct tlbe *stlbe)
538{
539 struct kvm_memory_slot *slot;
540 unsigned long pfn, hva;
541 int pfnmap = 0;
542 int tsize = BOOK3E_PAGESZ_4K;
543 struct tlbe_priv *priv;
544
545 /*
546 * Translate guest physical to true physical, acquiring
547 * a page reference if it is normal, non-reserved memory.
548 *
549 * gfn_to_memslot() must succeed because otherwise we wouldn't
550 * have gotten this far. Eventually we should just pass the slot
551 * pointer through from the first lookup.
552 */
553 slot = gfn_to_memslot(vcpu_e500->vcpu.kvm, gfn);
554 hva = gfn_to_hva_memslot(slot, gfn);
555
556 if (tlbsel == 1) {
557 struct vm_area_struct *vma;
558 down_read(&current->mm->mmap_sem);
559
560 vma = find_vma(current->mm, hva);
561 if (vma && hva >= vma->vm_start &&
562 (vma->vm_flags & VM_PFNMAP)) {
563 /*
564 * This VMA is a physically contiguous region (e.g.
565 * /dev/mem) that bypasses normal Linux page
566 * management. Find the overlap between the
567 * vma and the memslot.
568 */
569
570 unsigned long start, end;
571 unsigned long slot_start, slot_end;
572
573 pfnmap = 1;
574
575 start = vma->vm_pgoff;
576 end = start +
577 ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT);
578
579 pfn = start + ((hva - vma->vm_start) >> PAGE_SHIFT);
580
581 slot_start = pfn - (gfn - slot->base_gfn);
582 slot_end = slot_start + slot->npages;
583
584 if (start < slot_start)
585 start = slot_start;
586 if (end > slot_end)
587 end = slot_end;
588
589 tsize = (gtlbe->mas1 & MAS1_TSIZE_MASK) >>
590 MAS1_TSIZE_SHIFT;
591
592 /*
593 * e500 doesn't implement the lowest tsize bit,
594 * or 1K pages.
595 */
596 tsize = max(BOOK3E_PAGESZ_4K, tsize & ~1);
597
598 /*
599 * Now find the largest tsize (up to what the guest
600 * requested) that will cover gfn, stay within the
601 * range, and for which gfn and pfn are mutually
602 * aligned.
603 */
604
605 for (; tsize > BOOK3E_PAGESZ_4K; tsize -= 2) {
606 unsigned long gfn_start, gfn_end, tsize_pages;
607 tsize_pages = 1 << (tsize - 2);
608
609 gfn_start = gfn & ~(tsize_pages - 1);
610 gfn_end = gfn_start + tsize_pages;
611
612 if (gfn_start + pfn - gfn < start)
613 continue;
614 if (gfn_end + pfn - gfn > end)
615 continue;
616 if ((gfn & (tsize_pages - 1)) !=
617 (pfn & (tsize_pages - 1)))
618 continue;
619
620 gvaddr &= ~((tsize_pages << PAGE_SHIFT) - 1);
621 pfn &= ~(tsize_pages - 1);
622 break;
623 }
624 }
625
626 up_read(&current->mm->mmap_sem);
627 }
628
629 if (likely(!pfnmap)) {
630 pfn = gfn_to_pfn_memslot(vcpu_e500->vcpu.kvm, slot, gfn);
631 if (is_error_pfn(pfn)) {
632 printk(KERN_ERR "Couldn't get real page for gfn %lx!\n",
633 (long)gfn);
634 kvm_release_pfn_clean(pfn);
635 return;
636 }
637 }
638
639 /* Drop old priv and setup new one. */
640 priv = &vcpu_e500->gtlb_priv[tlbsel][esel];
641 kvmppc_e500_priv_release(priv);
642 kvmppc_e500_priv_setup(priv, gtlbe, pfn);
643
644 kvmppc_e500_setup_stlbe(vcpu_e500, gtlbe, tsize, priv, gvaddr, stlbe);
326} 645}
327 646
328/* XXX only map the one-one case, for now use TLB0 */ 647/* XXX only map the one-one case, for now use TLB0 */
329static int kvmppc_e500_stlbe_map(struct kvmppc_vcpu_e500 *vcpu_e500, 648static int kvmppc_e500_tlb0_map(struct kvmppc_vcpu_e500 *vcpu_e500,
330 int tlbsel, int esel) 649 int esel, struct tlbe *stlbe)
331{ 650{
332 struct tlbe *gtlbe; 651 struct tlbe *gtlbe;
333 652
334 gtlbe = &vcpu_e500->guest_tlb[tlbsel][esel]; 653 gtlbe = &vcpu_e500->gtlb_arch[0][esel];
335 654
336 kvmppc_e500_shadow_map(vcpu_e500, get_tlb_eaddr(gtlbe), 655 kvmppc_e500_shadow_map(vcpu_e500, get_tlb_eaddr(gtlbe),
337 get_tlb_raddr(gtlbe) >> PAGE_SHIFT, 656 get_tlb_raddr(gtlbe) >> PAGE_SHIFT,
338 gtlbe, tlbsel, esel); 657 gtlbe, 0, esel, stlbe);
339 658
340 return esel; 659 return esel;
341} 660}
@@ -344,53 +663,37 @@ static int kvmppc_e500_stlbe_map(struct kvmppc_vcpu_e500 *vcpu_e500,
344 * the shadow TLB. */ 663 * the shadow TLB. */
345/* XXX for both one-one and one-to-many , for now use TLB1 */ 664/* XXX for both one-one and one-to-many , for now use TLB1 */
346static int kvmppc_e500_tlb1_map(struct kvmppc_vcpu_e500 *vcpu_e500, 665static int kvmppc_e500_tlb1_map(struct kvmppc_vcpu_e500 *vcpu_e500,
347 u64 gvaddr, gfn_t gfn, struct tlbe *gtlbe) 666 u64 gvaddr, gfn_t gfn, struct tlbe *gtlbe, struct tlbe *stlbe)
348{ 667{
349 unsigned int victim; 668 unsigned int victim;
350 669
351 victim = vcpu_e500->guest_tlb_nv[1]++; 670 victim = vcpu_e500->gtlb_nv[1]++;
352 671
353 if (unlikely(vcpu_e500->guest_tlb_nv[1] >= tlb1_max_shadow_size())) 672 if (unlikely(vcpu_e500->gtlb_nv[1] >= tlb1_max_shadow_size()))
354 vcpu_e500->guest_tlb_nv[1] = 0; 673 vcpu_e500->gtlb_nv[1] = 0;
355 674
356 kvmppc_e500_shadow_map(vcpu_e500, gvaddr, gfn, gtlbe, 1, victim); 675 kvmppc_e500_shadow_map(vcpu_e500, gvaddr, gfn, gtlbe, 1, victim, stlbe);
357 676
358 return victim; 677 return victim;
359} 678}
360 679
361/* Invalidate all guest kernel mappings when enter usermode, 680void kvmppc_mmu_msr_notify(struct kvm_vcpu *vcpu, u32 old_msr)
362 * so that when they fault back in they will get the
363 * proper permission bits. */
364void kvmppc_mmu_priv_switch(struct kvm_vcpu *vcpu, int usermode)
365{ 681{
366 if (usermode) { 682 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
367 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
368 int i;
369
370 /* XXX Replace loop with fancy data structures. */
371 for (i = 0; i < tlb1_max_shadow_size(); i++)
372 kvmppc_e500_stlbe_invalidate(vcpu_e500, 1, i);
373 683
374 _tlbil_all(); 684 /* Recalc shadow pid since MSR changes */
375 } 685 kvmppc_e500_recalc_shadow_pid(vcpu_e500);
376} 686}
377 687
378static int kvmppc_e500_gtlbe_invalidate(struct kvmppc_vcpu_e500 *vcpu_e500, 688static inline int kvmppc_e500_gtlbe_invalidate(
379 int tlbsel, int esel) 689 struct kvmppc_vcpu_e500 *vcpu_e500,
690 int tlbsel, int esel)
380{ 691{
381 struct tlbe *gtlbe = &vcpu_e500->guest_tlb[tlbsel][esel]; 692 struct tlbe *gtlbe = &vcpu_e500->gtlb_arch[tlbsel][esel];
382 693
383 if (unlikely(get_tlb_iprot(gtlbe))) 694 if (unlikely(get_tlb_iprot(gtlbe)))
384 return -1; 695 return -1;
385 696
386 if (tlbsel == 1) {
387 kvmppc_e500_tlb1_invalidate(vcpu_e500, get_tlb_eaddr(gtlbe),
388 get_tlb_end(gtlbe),
389 get_tlb_tid(gtlbe));
390 } else {
391 kvmppc_e500_stlbe_invalidate(vcpu_e500, tlbsel, esel);
392 }
393
394 gtlbe->mas1 = 0; 697 gtlbe->mas1 = 0;
395 698
396 return 0; 699 return 0;
@@ -401,13 +704,14 @@ int kvmppc_e500_emul_mt_mmucsr0(struct kvmppc_vcpu_e500 *vcpu_e500, ulong value)
401 int esel; 704 int esel;
402 705
403 if (value & MMUCSR0_TLB0FI) 706 if (value & MMUCSR0_TLB0FI)
404 for (esel = 0; esel < vcpu_e500->guest_tlb_size[0]; esel++) 707 for (esel = 0; esel < vcpu_e500->gtlb_size[0]; esel++)
405 kvmppc_e500_gtlbe_invalidate(vcpu_e500, 0, esel); 708 kvmppc_e500_gtlbe_invalidate(vcpu_e500, 0, esel);
406 if (value & MMUCSR0_TLB1FI) 709 if (value & MMUCSR0_TLB1FI)
407 for (esel = 0; esel < vcpu_e500->guest_tlb_size[1]; esel++) 710 for (esel = 0; esel < vcpu_e500->gtlb_size[1]; esel++)
408 kvmppc_e500_gtlbe_invalidate(vcpu_e500, 1, esel); 711 kvmppc_e500_gtlbe_invalidate(vcpu_e500, 1, esel);
409 712
410 _tlbil_all(); 713 /* Invalidate all vcpu id mappings */
714 kvmppc_e500_id_table_reset_all(vcpu_e500);
411 715
412 return EMULATE_DONE; 716 return EMULATE_DONE;
413} 717}
@@ -428,7 +732,7 @@ int kvmppc_e500_emul_tlbivax(struct kvm_vcpu *vcpu, int ra, int rb)
428 732
429 if (ia) { 733 if (ia) {
430 /* invalidate all entries */ 734 /* invalidate all entries */
431 for (esel = 0; esel < vcpu_e500->guest_tlb_size[tlbsel]; esel++) 735 for (esel = 0; esel < vcpu_e500->gtlb_size[tlbsel]; esel++)
432 kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel); 736 kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel);
433 } else { 737 } else {
434 ea &= 0xfffff000; 738 ea &= 0xfffff000;
@@ -438,7 +742,8 @@ int kvmppc_e500_emul_tlbivax(struct kvm_vcpu *vcpu, int ra, int rb)
438 kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel); 742 kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel);
439 } 743 }
440 744
441 _tlbil_all(); 745 /* Invalidate all vcpu id mappings */
746 kvmppc_e500_id_table_reset_all(vcpu_e500);
442 747
443 return EMULATE_DONE; 748 return EMULATE_DONE;
444} 749}
@@ -452,9 +757,9 @@ int kvmppc_e500_emul_tlbre(struct kvm_vcpu *vcpu)
452 tlbsel = get_tlb_tlbsel(vcpu_e500); 757 tlbsel = get_tlb_tlbsel(vcpu_e500);
453 esel = get_tlb_esel(vcpu_e500, tlbsel); 758 esel = get_tlb_esel(vcpu_e500, tlbsel);
454 759
455 gtlbe = &vcpu_e500->guest_tlb[tlbsel][esel]; 760 gtlbe = &vcpu_e500->gtlb_arch[tlbsel][esel];
456 vcpu_e500->mas0 &= ~MAS0_NV(~0); 761 vcpu_e500->mas0 &= ~MAS0_NV(~0);
457 vcpu_e500->mas0 |= MAS0_NV(vcpu_e500->guest_tlb_nv[tlbsel]); 762 vcpu_e500->mas0 |= MAS0_NV(vcpu_e500->gtlb_nv[tlbsel]);
458 vcpu_e500->mas1 = gtlbe->mas1; 763 vcpu_e500->mas1 = gtlbe->mas1;
459 vcpu_e500->mas2 = gtlbe->mas2; 764 vcpu_e500->mas2 = gtlbe->mas2;
460 vcpu_e500->mas3 = gtlbe->mas3; 765 vcpu_e500->mas3 = gtlbe->mas3;
@@ -477,14 +782,14 @@ int kvmppc_e500_emul_tlbsx(struct kvm_vcpu *vcpu, int rb)
477 for (tlbsel = 0; tlbsel < 2; tlbsel++) { 782 for (tlbsel = 0; tlbsel < 2; tlbsel++) {
478 esel = kvmppc_e500_tlb_index(vcpu_e500, ea, tlbsel, pid, as); 783 esel = kvmppc_e500_tlb_index(vcpu_e500, ea, tlbsel, pid, as);
479 if (esel >= 0) { 784 if (esel >= 0) {
480 gtlbe = &vcpu_e500->guest_tlb[tlbsel][esel]; 785 gtlbe = &vcpu_e500->gtlb_arch[tlbsel][esel];
481 break; 786 break;
482 } 787 }
483 } 788 }
484 789
485 if (gtlbe) { 790 if (gtlbe) {
486 vcpu_e500->mas0 = MAS0_TLBSEL(tlbsel) | MAS0_ESEL(esel) 791 vcpu_e500->mas0 = MAS0_TLBSEL(tlbsel) | MAS0_ESEL(esel)
487 | MAS0_NV(vcpu_e500->guest_tlb_nv[tlbsel]); 792 | MAS0_NV(vcpu_e500->gtlb_nv[tlbsel]);
488 vcpu_e500->mas1 = gtlbe->mas1; 793 vcpu_e500->mas1 = gtlbe->mas1;
489 vcpu_e500->mas2 = gtlbe->mas2; 794 vcpu_e500->mas2 = gtlbe->mas2;
490 vcpu_e500->mas3 = gtlbe->mas3; 795 vcpu_e500->mas3 = gtlbe->mas3;
@@ -497,7 +802,7 @@ int kvmppc_e500_emul_tlbsx(struct kvm_vcpu *vcpu, int rb)
497 victim = (tlbsel == 0) ? tlb0_get_next_victim(vcpu_e500) : 0; 802 victim = (tlbsel == 0) ? tlb0_get_next_victim(vcpu_e500) : 0;
498 803
499 vcpu_e500->mas0 = MAS0_TLBSEL(tlbsel) | MAS0_ESEL(victim) 804 vcpu_e500->mas0 = MAS0_TLBSEL(tlbsel) | MAS0_ESEL(victim)
500 | MAS0_NV(vcpu_e500->guest_tlb_nv[tlbsel]); 805 | MAS0_NV(vcpu_e500->gtlb_nv[tlbsel]);
501 vcpu_e500->mas1 = (vcpu_e500->mas6 & MAS6_SPID0) 806 vcpu_e500->mas1 = (vcpu_e500->mas6 & MAS6_SPID0)
502 | (vcpu_e500->mas6 & (MAS6_SAS ? MAS1_TS : 0)) 807 | (vcpu_e500->mas6 & (MAS6_SAS ? MAS1_TS : 0))
503 | (vcpu_e500->mas4 & MAS4_TSIZED(~0)); 808 | (vcpu_e500->mas4 & MAS4_TSIZED(~0));
@@ -514,23 +819,16 @@ int kvmppc_e500_emul_tlbsx(struct kvm_vcpu *vcpu, int rb)
514int kvmppc_e500_emul_tlbwe(struct kvm_vcpu *vcpu) 819int kvmppc_e500_emul_tlbwe(struct kvm_vcpu *vcpu)
515{ 820{
516 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 821 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
517 u64 eaddr;
518 u64 raddr;
519 u32 tid;
520 struct tlbe *gtlbe; 822 struct tlbe *gtlbe;
521 int tlbsel, esel, stlbsel, sesel; 823 int tlbsel, esel;
522 824
523 tlbsel = get_tlb_tlbsel(vcpu_e500); 825 tlbsel = get_tlb_tlbsel(vcpu_e500);
524 esel = get_tlb_esel(vcpu_e500, tlbsel); 826 esel = get_tlb_esel(vcpu_e500, tlbsel);
525 827
526 gtlbe = &vcpu_e500->guest_tlb[tlbsel][esel]; 828 gtlbe = &vcpu_e500->gtlb_arch[tlbsel][esel];
527 829
528 if (get_tlb_v(gtlbe) && tlbsel == 1) { 830 if (get_tlb_v(gtlbe))
529 eaddr = get_tlb_eaddr(gtlbe); 831 kvmppc_e500_stlbe_invalidate(vcpu_e500, tlbsel, esel);
530 tid = get_tlb_tid(gtlbe);
531 kvmppc_e500_tlb1_invalidate(vcpu_e500, eaddr,
532 get_tlb_end(gtlbe), tid);
533 }
534 832
535 gtlbe->mas1 = vcpu_e500->mas1; 833 gtlbe->mas1 = vcpu_e500->mas1;
536 gtlbe->mas2 = vcpu_e500->mas2; 834 gtlbe->mas2 = vcpu_e500->mas2;
@@ -542,6 +840,12 @@ int kvmppc_e500_emul_tlbwe(struct kvm_vcpu *vcpu)
542 840
543 /* Invalidate shadow mappings for the about-to-be-clobbered TLBE. */ 841 /* Invalidate shadow mappings for the about-to-be-clobbered TLBE. */
544 if (tlbe_is_host_safe(vcpu, gtlbe)) { 842 if (tlbe_is_host_safe(vcpu, gtlbe)) {
843 struct tlbe stlbe;
844 int stlbsel, sesel;
845 u64 eaddr;
846 u64 raddr;
847
848 preempt_disable();
545 switch (tlbsel) { 849 switch (tlbsel) {
546 case 0: 850 case 0:
547 /* TLB0 */ 851 /* TLB0 */
@@ -549,7 +853,7 @@ int kvmppc_e500_emul_tlbwe(struct kvm_vcpu *vcpu)
549 gtlbe->mas1 |= MAS1_TSIZE(BOOK3E_PAGESZ_4K); 853 gtlbe->mas1 |= MAS1_TSIZE(BOOK3E_PAGESZ_4K);
550 854
551 stlbsel = 0; 855 stlbsel = 0;
552 sesel = kvmppc_e500_stlbe_map(vcpu_e500, 0, esel); 856 sesel = kvmppc_e500_tlb0_map(vcpu_e500, esel, &stlbe);
553 857
554 break; 858 break;
555 859
@@ -564,13 +868,14 @@ int kvmppc_e500_emul_tlbwe(struct kvm_vcpu *vcpu)
564 * are mapped on the fly. */ 868 * are mapped on the fly. */
565 stlbsel = 1; 869 stlbsel = 1;
566 sesel = kvmppc_e500_tlb1_map(vcpu_e500, eaddr, 870 sesel = kvmppc_e500_tlb1_map(vcpu_e500, eaddr,
567 raddr >> PAGE_SHIFT, gtlbe); 871 raddr >> PAGE_SHIFT, gtlbe, &stlbe);
568 break; 872 break;
569 873
570 default: 874 default:
571 BUG(); 875 BUG();
572 } 876 }
573 write_host_tlbe(vcpu_e500, stlbsel, sesel); 877 write_host_tlbe(vcpu_e500, stlbsel, sesel, &stlbe);
878 preempt_enable();
574 } 879 }
575 880
576 kvmppc_set_exit_type(vcpu, EMULATED_TLBWE_EXITS); 881 kvmppc_set_exit_type(vcpu, EMULATED_TLBWE_EXITS);
@@ -610,7 +915,7 @@ gpa_t kvmppc_mmu_xlate(struct kvm_vcpu *vcpu, unsigned int index,
610{ 915{
611 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 916 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
612 struct tlbe *gtlbe = 917 struct tlbe *gtlbe =
613 &vcpu_e500->guest_tlb[tlbsel_of(index)][esel_of(index)]; 918 &vcpu_e500->gtlb_arch[tlbsel_of(index)][esel_of(index)];
614 u64 pgmask = get_tlb_bytes(gtlbe) - 1; 919 u64 pgmask = get_tlb_bytes(gtlbe) - 1;
615 920
616 return get_tlb_raddr(gtlbe) | (eaddr & pgmask); 921 return get_tlb_raddr(gtlbe) | (eaddr & pgmask);
@@ -618,38 +923,37 @@ gpa_t kvmppc_mmu_xlate(struct kvm_vcpu *vcpu, unsigned int index,
618 923
619void kvmppc_mmu_destroy(struct kvm_vcpu *vcpu) 924void kvmppc_mmu_destroy(struct kvm_vcpu *vcpu)
620{ 925{
621 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
622 int tlbsel, i;
623
624 for (tlbsel = 0; tlbsel < 2; tlbsel++)
625 for (i = 0; i < vcpu_e500->guest_tlb_size[tlbsel]; i++)
626 kvmppc_e500_shadow_release(vcpu_e500, tlbsel, i);
627
628 /* discard all guest mapping */
629 _tlbil_all();
630} 926}
631 927
632void kvmppc_mmu_map(struct kvm_vcpu *vcpu, u64 eaddr, gpa_t gpaddr, 928void kvmppc_mmu_map(struct kvm_vcpu *vcpu, u64 eaddr, gpa_t gpaddr,
633 unsigned int index) 929 unsigned int index)
634{ 930{
635 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 931 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
932 struct tlbe_priv *priv;
933 struct tlbe *gtlbe, stlbe;
636 int tlbsel = tlbsel_of(index); 934 int tlbsel = tlbsel_of(index);
637 int esel = esel_of(index); 935 int esel = esel_of(index);
638 int stlbsel, sesel; 936 int stlbsel, sesel;
639 937
938 gtlbe = &vcpu_e500->gtlb_arch[tlbsel][esel];
939
940 preempt_disable();
640 switch (tlbsel) { 941 switch (tlbsel) {
641 case 0: 942 case 0:
642 stlbsel = 0; 943 stlbsel = 0;
643 sesel = esel; 944 sesel = esel;
945 priv = &vcpu_e500->gtlb_priv[stlbsel][sesel];
946
947 kvmppc_e500_setup_stlbe(vcpu_e500, gtlbe, BOOK3E_PAGESZ_4K,
948 priv, eaddr, &stlbe);
644 break; 949 break;
645 950
646 case 1: { 951 case 1: {
647 gfn_t gfn = gpaddr >> PAGE_SHIFT; 952 gfn_t gfn = gpaddr >> PAGE_SHIFT;
648 struct tlbe *gtlbe
649 = &vcpu_e500->guest_tlb[tlbsel][esel];
650 953
651 stlbsel = 1; 954 stlbsel = 1;
652 sesel = kvmppc_e500_tlb1_map(vcpu_e500, eaddr, gfn, gtlbe); 955 sesel = kvmppc_e500_tlb1_map(vcpu_e500, eaddr, gfn,
956 gtlbe, &stlbe);
653 break; 957 break;
654 } 958 }
655 959
@@ -657,7 +961,9 @@ void kvmppc_mmu_map(struct kvm_vcpu *vcpu, u64 eaddr, gpa_t gpaddr,
657 BUG(); 961 BUG();
658 break; 962 break;
659 } 963 }
660 write_host_tlbe(vcpu_e500, stlbsel, sesel); 964
965 write_host_tlbe(vcpu_e500, stlbsel, sesel, &stlbe);
966 preempt_enable();
661} 967}
662 968
663int kvmppc_e500_tlb_search(struct kvm_vcpu *vcpu, 969int kvmppc_e500_tlb_search(struct kvm_vcpu *vcpu,
@@ -679,8 +985,10 @@ void kvmppc_set_pid(struct kvm_vcpu *vcpu, u32 pid)
679{ 985{
680 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); 986 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
681 987
682 vcpu_e500->pid[0] = vcpu->arch.shadow_pid = 988 if (vcpu->arch.pid != pid) {
683 vcpu->arch.pid = pid; 989 vcpu_e500->pid[0] = vcpu->arch.pid = pid;
990 kvmppc_e500_recalc_shadow_pid(vcpu_e500);
991 }
684} 992}
685 993
686void kvmppc_e500_tlb_setup(struct kvmppc_vcpu_e500 *vcpu_e500) 994void kvmppc_e500_tlb_setup(struct kvmppc_vcpu_e500 *vcpu_e500)
@@ -688,14 +996,14 @@ void kvmppc_e500_tlb_setup(struct kvmppc_vcpu_e500 *vcpu_e500)
688 struct tlbe *tlbe; 996 struct tlbe *tlbe;
689 997
690 /* Insert large initial mapping for guest. */ 998 /* Insert large initial mapping for guest. */
691 tlbe = &vcpu_e500->guest_tlb[1][0]; 999 tlbe = &vcpu_e500->gtlb_arch[1][0];
692 tlbe->mas1 = MAS1_VALID | MAS1_TSIZE(BOOK3E_PAGESZ_256M); 1000 tlbe->mas1 = MAS1_VALID | MAS1_TSIZE(BOOK3E_PAGESZ_256M);
693 tlbe->mas2 = 0; 1001 tlbe->mas2 = 0;
694 tlbe->mas3 = E500_TLB_SUPER_PERM_MASK; 1002 tlbe->mas3 = E500_TLB_SUPER_PERM_MASK;
695 tlbe->mas7 = 0; 1003 tlbe->mas7 = 0;
696 1004
697 /* 4K map for serial output. Used by kernel wrapper. */ 1005 /* 4K map for serial output. Used by kernel wrapper. */
698 tlbe = &vcpu_e500->guest_tlb[1][1]; 1006 tlbe = &vcpu_e500->gtlb_arch[1][1];
699 tlbe->mas1 = MAS1_VALID | MAS1_TSIZE(BOOK3E_PAGESZ_4K); 1007 tlbe->mas1 = MAS1_VALID | MAS1_TSIZE(BOOK3E_PAGESZ_4K);
700 tlbe->mas2 = (0xe0004500 & 0xFFFFF000) | MAS2_I | MAS2_G; 1008 tlbe->mas2 = (0xe0004500 & 0xFFFFF000) | MAS2_I | MAS2_G;
701 tlbe->mas3 = (0xe0004500 & 0xFFFFF000) | E500_TLB_SUPER_PERM_MASK; 1009 tlbe->mas3 = (0xe0004500 & 0xFFFFF000) | E500_TLB_SUPER_PERM_MASK;
@@ -706,68 +1014,64 @@ int kvmppc_e500_tlb_init(struct kvmppc_vcpu_e500 *vcpu_e500)
706{ 1014{
707 tlb1_entry_num = mfspr(SPRN_TLB1CFG) & 0xFFF; 1015 tlb1_entry_num = mfspr(SPRN_TLB1CFG) & 0xFFF;
708 1016
709 vcpu_e500->guest_tlb_size[0] = KVM_E500_TLB0_SIZE; 1017 vcpu_e500->gtlb_size[0] = KVM_E500_TLB0_SIZE;
710 vcpu_e500->guest_tlb[0] = 1018 vcpu_e500->gtlb_arch[0] =
711 kzalloc(sizeof(struct tlbe) * KVM_E500_TLB0_SIZE, GFP_KERNEL); 1019 kzalloc(sizeof(struct tlbe) * KVM_E500_TLB0_SIZE, GFP_KERNEL);
712 if (vcpu_e500->guest_tlb[0] == NULL) 1020 if (vcpu_e500->gtlb_arch[0] == NULL)
713 goto err_out; 1021 goto err_out;
714 1022
715 vcpu_e500->shadow_tlb_size[0] = KVM_E500_TLB0_SIZE; 1023 vcpu_e500->gtlb_size[1] = KVM_E500_TLB1_SIZE;
716 vcpu_e500->shadow_tlb[0] = 1024 vcpu_e500->gtlb_arch[1] =
717 kzalloc(sizeof(struct tlbe) * KVM_E500_TLB0_SIZE, GFP_KERNEL);
718 if (vcpu_e500->shadow_tlb[0] == NULL)
719 goto err_out_guest0;
720
721 vcpu_e500->guest_tlb_size[1] = KVM_E500_TLB1_SIZE;
722 vcpu_e500->guest_tlb[1] =
723 kzalloc(sizeof(struct tlbe) * KVM_E500_TLB1_SIZE, GFP_KERNEL); 1025 kzalloc(sizeof(struct tlbe) * KVM_E500_TLB1_SIZE, GFP_KERNEL);
724 if (vcpu_e500->guest_tlb[1] == NULL) 1026 if (vcpu_e500->gtlb_arch[1] == NULL)
725 goto err_out_shadow0; 1027 goto err_out_guest0;
726 1028
727 vcpu_e500->shadow_tlb_size[1] = tlb1_entry_num; 1029 vcpu_e500->gtlb_priv[0] = (struct tlbe_priv *)
728 vcpu_e500->shadow_tlb[1] = 1030 kzalloc(sizeof(struct tlbe_priv) * KVM_E500_TLB0_SIZE, GFP_KERNEL);
729 kzalloc(sizeof(struct tlbe) * tlb1_entry_num, GFP_KERNEL); 1031 if (vcpu_e500->gtlb_priv[0] == NULL)
730 if (vcpu_e500->shadow_tlb[1] == NULL)
731 goto err_out_guest1; 1032 goto err_out_guest1;
1033 vcpu_e500->gtlb_priv[1] = (struct tlbe_priv *)
1034 kzalloc(sizeof(struct tlbe_priv) * KVM_E500_TLB1_SIZE, GFP_KERNEL);
732 1035
733 vcpu_e500->shadow_pages[0] = (struct page **) 1036 if (vcpu_e500->gtlb_priv[1] == NULL)
734 kzalloc(sizeof(struct page *) * KVM_E500_TLB0_SIZE, GFP_KERNEL); 1037 goto err_out_priv0;
735 if (vcpu_e500->shadow_pages[0] == NULL)
736 goto err_out_shadow1;
737 1038
738 vcpu_e500->shadow_pages[1] = (struct page **) 1039 if (kvmppc_e500_id_table_alloc(vcpu_e500) == NULL)
739 kzalloc(sizeof(struct page *) * tlb1_entry_num, GFP_KERNEL); 1040 goto err_out_priv1;
740 if (vcpu_e500->shadow_pages[1] == NULL)
741 goto err_out_page0;
742 1041
743 /* Init TLB configuration register */ 1042 /* Init TLB configuration register */
744 vcpu_e500->tlb0cfg = mfspr(SPRN_TLB0CFG) & ~0xfffUL; 1043 vcpu_e500->tlb0cfg = mfspr(SPRN_TLB0CFG) & ~0xfffUL;
745 vcpu_e500->tlb0cfg |= vcpu_e500->guest_tlb_size[0]; 1044 vcpu_e500->tlb0cfg |= vcpu_e500->gtlb_size[0];
746 vcpu_e500->tlb1cfg = mfspr(SPRN_TLB1CFG) & ~0xfffUL; 1045 vcpu_e500->tlb1cfg = mfspr(SPRN_TLB1CFG) & ~0xfffUL;
747 vcpu_e500->tlb1cfg |= vcpu_e500->guest_tlb_size[1]; 1046 vcpu_e500->tlb1cfg |= vcpu_e500->gtlb_size[1];
748 1047
749 return 0; 1048 return 0;
750 1049
751err_out_page0: 1050err_out_priv1:
752 kfree(vcpu_e500->shadow_pages[0]); 1051 kfree(vcpu_e500->gtlb_priv[1]);
753err_out_shadow1: 1052err_out_priv0:
754 kfree(vcpu_e500->shadow_tlb[1]); 1053 kfree(vcpu_e500->gtlb_priv[0]);
755err_out_guest1: 1054err_out_guest1:
756 kfree(vcpu_e500->guest_tlb[1]); 1055 kfree(vcpu_e500->gtlb_arch[1]);
757err_out_shadow0:
758 kfree(vcpu_e500->shadow_tlb[0]);
759err_out_guest0: 1056err_out_guest0:
760 kfree(vcpu_e500->guest_tlb[0]); 1057 kfree(vcpu_e500->gtlb_arch[0]);
761err_out: 1058err_out:
762 return -1; 1059 return -1;
763} 1060}
764 1061
765void kvmppc_e500_tlb_uninit(struct kvmppc_vcpu_e500 *vcpu_e500) 1062void kvmppc_e500_tlb_uninit(struct kvmppc_vcpu_e500 *vcpu_e500)
766{ 1063{
767 kfree(vcpu_e500->shadow_pages[1]); 1064 int stlbsel, i;
768 kfree(vcpu_e500->shadow_pages[0]); 1065
769 kfree(vcpu_e500->shadow_tlb[1]); 1066 /* release all privs */
770 kfree(vcpu_e500->guest_tlb[1]); 1067 for (stlbsel = 0; stlbsel < 2; stlbsel++)
771 kfree(vcpu_e500->shadow_tlb[0]); 1068 for (i = 0; i < vcpu_e500->gtlb_size[stlbsel]; i++) {
772 kfree(vcpu_e500->guest_tlb[0]); 1069 struct tlbe_priv *priv =
1070 &vcpu_e500->gtlb_priv[stlbsel][i];
1071 kvmppc_e500_priv_release(priv);
1072 }
1073
1074 kvmppc_e500_id_table_free(vcpu_e500);
1075 kfree(vcpu_e500->gtlb_arch[1]);
1076 kfree(vcpu_e500->gtlb_arch[0]);
773} 1077}
diff --git a/arch/powerpc/kvm/e500_tlb.h b/arch/powerpc/kvm/e500_tlb.h
index 458946b4775d..59b88e99a235 100644
--- a/arch/powerpc/kvm/e500_tlb.h
+++ b/arch/powerpc/kvm/e500_tlb.h
@@ -1,5 +1,5 @@
1/* 1/*
2 * Copyright (C) 2008 Freescale Semiconductor, Inc. All rights reserved. 2 * Copyright (C) 2008-2011 Freescale Semiconductor, Inc. All rights reserved.
3 * 3 *
4 * Author: Yu Liu, yu.liu@freescale.com 4 * Author: Yu Liu, yu.liu@freescale.com
5 * 5 *
@@ -55,6 +55,7 @@ extern void kvmppc_e500_tlb_load(struct kvm_vcpu *, int);
55extern int kvmppc_e500_tlb_init(struct kvmppc_vcpu_e500 *); 55extern int kvmppc_e500_tlb_init(struct kvmppc_vcpu_e500 *);
56extern void kvmppc_e500_tlb_uninit(struct kvmppc_vcpu_e500 *); 56extern void kvmppc_e500_tlb_uninit(struct kvmppc_vcpu_e500 *);
57extern void kvmppc_e500_tlb_setup(struct kvmppc_vcpu_e500 *); 57extern void kvmppc_e500_tlb_setup(struct kvmppc_vcpu_e500 *);
58extern void kvmppc_e500_recalc_shadow_pid(struct kvmppc_vcpu_e500 *);
58 59
59/* TLB helper functions */ 60/* TLB helper functions */
60static inline unsigned int get_tlb_size(const struct tlbe *tlbe) 61static inline unsigned int get_tlb_size(const struct tlbe *tlbe)
@@ -110,6 +111,16 @@ static inline unsigned int get_cur_pid(struct kvm_vcpu *vcpu)
110 return vcpu->arch.pid & 0xff; 111 return vcpu->arch.pid & 0xff;
111} 112}
112 113
114static inline unsigned int get_cur_as(struct kvm_vcpu *vcpu)
115{
116 return !!(vcpu->arch.shared->msr & (MSR_IS | MSR_DS));
117}
118
119static inline unsigned int get_cur_pr(struct kvm_vcpu *vcpu)
120{
121 return !!(vcpu->arch.shared->msr & MSR_PR);
122}
123
113static inline unsigned int get_cur_spid( 124static inline unsigned int get_cur_spid(
114 const struct kvmppc_vcpu_e500 *vcpu_e500) 125 const struct kvmppc_vcpu_e500 *vcpu_e500)
115{ 126{
diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c
index 616dd516ca1f..a107c9be0fb1 100644
--- a/arch/powerpc/kvm/powerpc.c
+++ b/arch/powerpc/kvm/powerpc.c
@@ -30,6 +30,7 @@
30#include <asm/uaccess.h> 30#include <asm/uaccess.h>
31#include <asm/kvm_ppc.h> 31#include <asm/kvm_ppc.h>
32#include <asm/tlbflush.h> 32#include <asm/tlbflush.h>
33#include <asm/cputhreads.h>
33#include "timing.h" 34#include "timing.h"
34#include "../mm/mmu_decl.h" 35#include "../mm/mmu_decl.h"
35 36
@@ -38,8 +39,12 @@
38 39
39int kvm_arch_vcpu_runnable(struct kvm_vcpu *v) 40int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
40{ 41{
42#ifndef CONFIG_KVM_BOOK3S_64_HV
41 return !(v->arch.shared->msr & MSR_WE) || 43 return !(v->arch.shared->msr & MSR_WE) ||
42 !!(v->arch.pending_exceptions); 44 !!(v->arch.pending_exceptions);
45#else
46 return !(v->arch.ceded) || !!(v->arch.pending_exceptions);
47#endif
43} 48}
44 49
45int kvmppc_kvm_pv(struct kvm_vcpu *vcpu) 50int kvmppc_kvm_pv(struct kvm_vcpu *vcpu)
@@ -73,7 +78,8 @@ int kvmppc_kvm_pv(struct kvm_vcpu *vcpu)
73 } 78 }
74 case HC_VENDOR_KVM | KVM_HC_FEATURES: 79 case HC_VENDOR_KVM | KVM_HC_FEATURES:
75 r = HC_EV_SUCCESS; 80 r = HC_EV_SUCCESS;
76#if defined(CONFIG_PPC_BOOK3S) /* XXX Missing magic page on BookE */ 81#if defined(CONFIG_PPC_BOOK3S) || defined(CONFIG_KVM_E500)
82 /* XXX Missing magic page on 44x */
77 r2 |= (1 << KVM_FEATURE_MAGIC_PAGE); 83 r2 |= (1 << KVM_FEATURE_MAGIC_PAGE);
78#endif 84#endif
79 85
@@ -147,7 +153,7 @@ void kvm_arch_check_processor_compat(void *rtn)
147 153
148int kvm_arch_init_vm(struct kvm *kvm) 154int kvm_arch_init_vm(struct kvm *kvm)
149{ 155{
150 return 0; 156 return kvmppc_core_init_vm(kvm);
151} 157}
152 158
153void kvm_arch_destroy_vm(struct kvm *kvm) 159void kvm_arch_destroy_vm(struct kvm *kvm)
@@ -163,6 +169,9 @@ void kvm_arch_destroy_vm(struct kvm *kvm)
163 kvm->vcpus[i] = NULL; 169 kvm->vcpus[i] = NULL;
164 170
165 atomic_set(&kvm->online_vcpus, 0); 171 atomic_set(&kvm->online_vcpus, 0);
172
173 kvmppc_core_destroy_vm(kvm);
174
166 mutex_unlock(&kvm->lock); 175 mutex_unlock(&kvm->lock);
167} 176}
168 177
@@ -180,10 +189,13 @@ int kvm_dev_ioctl_check_extension(long ext)
180#else 189#else
181 case KVM_CAP_PPC_SEGSTATE: 190 case KVM_CAP_PPC_SEGSTATE:
182#endif 191#endif
183 case KVM_CAP_PPC_PAIRED_SINGLES:
184 case KVM_CAP_PPC_UNSET_IRQ: 192 case KVM_CAP_PPC_UNSET_IRQ:
185 case KVM_CAP_PPC_IRQ_LEVEL: 193 case KVM_CAP_PPC_IRQ_LEVEL:
186 case KVM_CAP_ENABLE_CAP: 194 case KVM_CAP_ENABLE_CAP:
195 r = 1;
196 break;
197#ifndef CONFIG_KVM_BOOK3S_64_HV
198 case KVM_CAP_PPC_PAIRED_SINGLES:
187 case KVM_CAP_PPC_OSI: 199 case KVM_CAP_PPC_OSI:
188 case KVM_CAP_PPC_GET_PVINFO: 200 case KVM_CAP_PPC_GET_PVINFO:
189 r = 1; 201 r = 1;
@@ -191,6 +203,21 @@ int kvm_dev_ioctl_check_extension(long ext)
191 case KVM_CAP_COALESCED_MMIO: 203 case KVM_CAP_COALESCED_MMIO:
192 r = KVM_COALESCED_MMIO_PAGE_OFFSET; 204 r = KVM_COALESCED_MMIO_PAGE_OFFSET;
193 break; 205 break;
206#endif
207#ifdef CONFIG_KVM_BOOK3S_64_HV
208 case KVM_CAP_SPAPR_TCE:
209 r = 1;
210 break;
211 case KVM_CAP_PPC_SMT:
212 r = threads_per_core;
213 break;
214 case KVM_CAP_PPC_RMA:
215 r = 1;
216 /* PPC970 requires an RMA */
217 if (cpu_has_feature(CPU_FTR_ARCH_201))
218 r = 2;
219 break;
220#endif
194 default: 221 default:
195 r = 0; 222 r = 0;
196 break; 223 break;
@@ -211,7 +238,7 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
211 struct kvm_userspace_memory_region *mem, 238 struct kvm_userspace_memory_region *mem,
212 int user_alloc) 239 int user_alloc)
213{ 240{
214 return 0; 241 return kvmppc_core_prepare_memory_region(kvm, mem);
215} 242}
216 243
217void kvm_arch_commit_memory_region(struct kvm *kvm, 244void kvm_arch_commit_memory_region(struct kvm *kvm,
@@ -219,7 +246,7 @@ void kvm_arch_commit_memory_region(struct kvm *kvm,
219 struct kvm_memory_slot old, 246 struct kvm_memory_slot old,
220 int user_alloc) 247 int user_alloc)
221{ 248{
222 return; 249 kvmppc_core_commit_memory_region(kvm, mem);
223} 250}
224 251
225 252
@@ -287,6 +314,7 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
287 hrtimer_init(&vcpu->arch.dec_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS); 314 hrtimer_init(&vcpu->arch.dec_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS);
288 tasklet_init(&vcpu->arch.tasklet, kvmppc_decrementer_func, (ulong)vcpu); 315 tasklet_init(&vcpu->arch.tasklet, kvmppc_decrementer_func, (ulong)vcpu);
289 vcpu->arch.dec_timer.function = kvmppc_decrementer_wakeup; 316 vcpu->arch.dec_timer.function = kvmppc_decrementer_wakeup;
317 vcpu->arch.dec_expires = ~(u64)0;
290 318
291#ifdef CONFIG_KVM_EXIT_TIMING 319#ifdef CONFIG_KVM_EXIT_TIMING
292 mutex_init(&vcpu->arch.exit_timing_lock); 320 mutex_init(&vcpu->arch.exit_timing_lock);
@@ -313,6 +341,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
313 mtspr(SPRN_VRSAVE, vcpu->arch.vrsave); 341 mtspr(SPRN_VRSAVE, vcpu->arch.vrsave);
314#endif 342#endif
315 kvmppc_core_vcpu_load(vcpu, cpu); 343 kvmppc_core_vcpu_load(vcpu, cpu);
344 vcpu->cpu = smp_processor_id();
316} 345}
317 346
318void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 347void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
@@ -321,6 +350,7 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
321#ifdef CONFIG_BOOKE 350#ifdef CONFIG_BOOKE
322 vcpu->arch.vrsave = mfspr(SPRN_VRSAVE); 351 vcpu->arch.vrsave = mfspr(SPRN_VRSAVE);
323#endif 352#endif
353 vcpu->cpu = -1;
324} 354}
325 355
326int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 356int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
@@ -492,15 +522,18 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
492 for (i = 0; i < 32; i++) 522 for (i = 0; i < 32; i++)
493 kvmppc_set_gpr(vcpu, i, gprs[i]); 523 kvmppc_set_gpr(vcpu, i, gprs[i]);
494 vcpu->arch.osi_needed = 0; 524 vcpu->arch.osi_needed = 0;
525 } else if (vcpu->arch.hcall_needed) {
526 int i;
527
528 kvmppc_set_gpr(vcpu, 3, run->papr_hcall.ret);
529 for (i = 0; i < 9; ++i)
530 kvmppc_set_gpr(vcpu, 4 + i, run->papr_hcall.args[i]);
531 vcpu->arch.hcall_needed = 0;
495 } 532 }
496 533
497 kvmppc_core_deliver_interrupts(vcpu); 534 kvmppc_core_deliver_interrupts(vcpu);
498 535
499 local_irq_disable(); 536 r = kvmppc_vcpu_run(run, vcpu);
500 kvm_guest_enter();
501 r = __kvmppc_vcpu_run(run, vcpu);
502 kvm_guest_exit();
503 local_irq_enable();
504 537
505 if (vcpu->sigset_active) 538 if (vcpu->sigset_active)
506 sigprocmask(SIG_SETMASK, &sigsaved, NULL); 539 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
@@ -518,6 +551,8 @@ int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, struct kvm_interrupt *irq)
518 if (waitqueue_active(&vcpu->wq)) { 551 if (waitqueue_active(&vcpu->wq)) {
519 wake_up_interruptible(&vcpu->wq); 552 wake_up_interruptible(&vcpu->wq);
520 vcpu->stat.halt_wakeup++; 553 vcpu->stat.halt_wakeup++;
554 } else if (vcpu->cpu != -1) {
555 smp_send_reschedule(vcpu->cpu);
521 } 556 }
522 557
523 return 0; 558 return 0;
@@ -633,6 +668,29 @@ long kvm_arch_vm_ioctl(struct file *filp,
633 668
634 break; 669 break;
635 } 670 }
671#ifdef CONFIG_KVM_BOOK3S_64_HV
672 case KVM_CREATE_SPAPR_TCE: {
673 struct kvm_create_spapr_tce create_tce;
674 struct kvm *kvm = filp->private_data;
675
676 r = -EFAULT;
677 if (copy_from_user(&create_tce, argp, sizeof(create_tce)))
678 goto out;
679 r = kvm_vm_ioctl_create_spapr_tce(kvm, &create_tce);
680 goto out;
681 }
682
683 case KVM_ALLOCATE_RMA: {
684 struct kvm *kvm = filp->private_data;
685 struct kvm_allocate_rma rma;
686
687 r = kvm_vm_ioctl_allocate_rma(kvm, &rma);
688 if (r >= 0 && copy_to_user(argp, &rma, sizeof(rma)))
689 r = -EFAULT;
690 break;
691 }
692#endif /* CONFIG_KVM_BOOK3S_64_HV */
693
636 default: 694 default:
637 r = -ENOTTY; 695 r = -ENOTTY;
638 } 696 }
diff --git a/arch/powerpc/kvm/timing.c b/arch/powerpc/kvm/timing.c
index 319177df9587..07b6110a4bb7 100644
--- a/arch/powerpc/kvm/timing.c
+++ b/arch/powerpc/kvm/timing.c
@@ -56,15 +56,6 @@ static void add_exit_timing(struct kvm_vcpu *vcpu, u64 duration, int type)
56{ 56{
57 u64 old; 57 u64 old;
58 58
59 do_div(duration, tb_ticks_per_usec);
60 if (unlikely(duration > 0xFFFFFFFF)) {
61 printk(KERN_ERR"%s - duration too big -> overflow"
62 " duration %lld type %d exit #%d\n",
63 __func__, duration, type,
64 vcpu->arch.timing_count_type[type]);
65 return;
66 }
67
68 mutex_lock(&vcpu->arch.exit_timing_lock); 59 mutex_lock(&vcpu->arch.exit_timing_lock);
69 60
70 vcpu->arch.timing_count_type[type]++; 61 vcpu->arch.timing_count_type[type]++;
diff --git a/arch/powerpc/kvm/trace.h b/arch/powerpc/kvm/trace.h
index 3aca1b042b8c..b135d3d397db 100644
--- a/arch/powerpc/kvm/trace.h
+++ b/arch/powerpc/kvm/trace.h
@@ -103,7 +103,7 @@ TRACE_EVENT(kvm_gtlb_write,
103 * Book3S trace points * 103 * Book3S trace points *
104 *************************************************************************/ 104 *************************************************************************/
105 105
106#ifdef CONFIG_PPC_BOOK3S 106#ifdef CONFIG_KVM_BOOK3S_PR
107 107
108TRACE_EVENT(kvm_book3s_exit, 108TRACE_EVENT(kvm_book3s_exit,
109 TP_PROTO(unsigned int exit_nr, struct kvm_vcpu *vcpu), 109 TP_PROTO(unsigned int exit_nr, struct kvm_vcpu *vcpu),
@@ -252,7 +252,7 @@ TRACE_EVENT(kvm_book3s_mmu_flush,
252 ), 252 ),
253 253
254 TP_fast_assign( 254 TP_fast_assign(
255 __entry->count = vcpu->arch.hpte_cache_count; 255 __entry->count = to_book3s(vcpu)->hpte_cache_count;
256 __entry->p1 = p1; 256 __entry->p1 = p1;
257 __entry->p2 = p2; 257 __entry->p2 = p2;
258 __entry->type = type; 258 __entry->type = type;
diff --git a/arch/powerpc/mm/hash_native_64.c b/arch/powerpc/mm/hash_native_64.c
index dfd764896db0..90039bc64119 100644
--- a/arch/powerpc/mm/hash_native_64.c
+++ b/arch/powerpc/mm/hash_native_64.c
@@ -37,7 +37,7 @@
37 37
38#define HPTE_LOCK_BIT 3 38#define HPTE_LOCK_BIT 3
39 39
40static DEFINE_RAW_SPINLOCK(native_tlbie_lock); 40DEFINE_RAW_SPINLOCK(native_tlbie_lock);
41 41
42static inline void __tlbie(unsigned long va, int psize, int ssize) 42static inline void __tlbie(unsigned long va, int psize, int ssize)
43{ 43{
@@ -51,7 +51,7 @@ static inline void __tlbie(unsigned long va, int psize, int ssize)
51 va &= ~0xffful; 51 va &= ~0xffful;
52 va |= ssize << 8; 52 va |= ssize << 8;
53 asm volatile(ASM_FTR_IFCLR("tlbie %0,0", PPC_TLBIE(%1,%0), %2) 53 asm volatile(ASM_FTR_IFCLR("tlbie %0,0", PPC_TLBIE(%1,%0), %2)
54 : : "r" (va), "r"(0), "i" (CPU_FTR_HVMODE_206) 54 : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206)
55 : "memory"); 55 : "memory");
56 break; 56 break;
57 default: 57 default:
@@ -61,7 +61,7 @@ static inline void __tlbie(unsigned long va, int psize, int ssize)
61 va |= ssize << 8; 61 va |= ssize << 8;
62 va |= 1; /* L */ 62 va |= 1; /* L */
63 asm volatile(ASM_FTR_IFCLR("tlbie %0,1", PPC_TLBIE(%1,%0), %2) 63 asm volatile(ASM_FTR_IFCLR("tlbie %0,1", PPC_TLBIE(%1,%0), %2)
64 : : "r" (va), "r"(0), "i" (CPU_FTR_HVMODE_206) 64 : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206)
65 : "memory"); 65 : "memory");
66 break; 66 break;
67 } 67 }
diff --git a/arch/powerpc/platforms/iseries/exception.S b/arch/powerpc/platforms/iseries/exception.S
index 29c02f36b32f..f519ee17ff7d 100644
--- a/arch/powerpc/platforms/iseries/exception.S
+++ b/arch/powerpc/platforms/iseries/exception.S
@@ -167,7 +167,7 @@ BEGIN_FTR_SECTION
167 std r12,PACA_EXGEN+EX_R13(r13) 167 std r12,PACA_EXGEN+EX_R13(r13)
168 EXCEPTION_PROLOG_ISERIES_1 168 EXCEPTION_PROLOG_ISERIES_1
169FTR_SECTION_ELSE 169FTR_SECTION_ELSE
170 EXCEPTION_PROLOG_1(PACA_EXGEN) 170 EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, 0)
171 EXCEPTION_PROLOG_ISERIES_1 171 EXCEPTION_PROLOG_ISERIES_1
172ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_SLB) 172ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_SLB)
173 b data_access_common 173 b data_access_common
diff --git a/arch/powerpc/platforms/iseries/exception.h b/arch/powerpc/platforms/iseries/exception.h
index bae3fba5ad8e..50271b550a99 100644
--- a/arch/powerpc/platforms/iseries/exception.h
+++ b/arch/powerpc/platforms/iseries/exception.h
@@ -39,7 +39,7 @@
39label##_iSeries: \ 39label##_iSeries: \
40 HMT_MEDIUM; \ 40 HMT_MEDIUM; \
41 mtspr SPRN_SPRG_SCRATCH0,r13; /* save r13 */ \ 41 mtspr SPRN_SPRG_SCRATCH0,r13; /* save r13 */ \
42 EXCEPTION_PROLOG_1(area); \ 42 EXCEPTION_PROLOG_1(area, NOTEST, 0); \
43 EXCEPTION_PROLOG_ISERIES_1; \ 43 EXCEPTION_PROLOG_ISERIES_1; \
44 b label##_common 44 b label##_common
45 45
@@ -48,7 +48,7 @@ label##_iSeries: \
48label##_iSeries: \ 48label##_iSeries: \
49 HMT_MEDIUM; \ 49 HMT_MEDIUM; \
50 mtspr SPRN_SPRG_SCRATCH0,r13; /* save r13 */ \ 50 mtspr SPRN_SPRG_SCRATCH0,r13; /* save r13 */ \
51 EXCEPTION_PROLOG_1(PACA_EXGEN); \ 51 EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, 0); \
52 lbz r10,PACASOFTIRQEN(r13); \ 52 lbz r10,PACASOFTIRQEN(r13); \
53 cmpwi 0,r10,0; \ 53 cmpwi 0,r10,0; \
54 beq- label##_iSeries_masked; \ 54 beq- label##_iSeries_masked; \
diff --git a/arch/powerpc/sysdev/xics/icp-native.c b/arch/powerpc/sysdev/xics/icp-native.c
index 1f15ad436140..ba382b59b926 100644
--- a/arch/powerpc/sysdev/xics/icp-native.c
+++ b/arch/powerpc/sysdev/xics/icp-native.c
@@ -17,6 +17,7 @@
17#include <linux/cpu.h> 17#include <linux/cpu.h>
18#include <linux/of.h> 18#include <linux/of.h>
19#include <linux/spinlock.h> 19#include <linux/spinlock.h>
20#include <linux/module.h>
20 21
21#include <asm/prom.h> 22#include <asm/prom.h>
22#include <asm/io.h> 23#include <asm/io.h>
@@ -24,6 +25,7 @@
24#include <asm/irq.h> 25#include <asm/irq.h>
25#include <asm/errno.h> 26#include <asm/errno.h>
26#include <asm/xics.h> 27#include <asm/xics.h>
28#include <asm/kvm_ppc.h>
27 29
28struct icp_ipl { 30struct icp_ipl {
29 union { 31 union {
@@ -139,6 +141,12 @@ static void icp_native_cause_ipi(int cpu, unsigned long data)
139 icp_native_set_qirr(cpu, IPI_PRIORITY); 141 icp_native_set_qirr(cpu, IPI_PRIORITY);
140} 142}
141 143
144void xics_wake_cpu(int cpu)
145{
146 icp_native_set_qirr(cpu, IPI_PRIORITY);
147}
148EXPORT_SYMBOL_GPL(xics_wake_cpu);
149
142static irqreturn_t icp_native_ipi_action(int irq, void *dev_id) 150static irqreturn_t icp_native_ipi_action(int irq, void *dev_id)
143{ 151{
144 int cpu = smp_processor_id(); 152 int cpu = smp_processor_id();
@@ -185,6 +193,7 @@ static int __init icp_native_map_one_cpu(int hw_id, unsigned long addr,
185 } 193 }
186 194
187 icp_native_regs[cpu] = ioremap(addr, size); 195 icp_native_regs[cpu] = ioremap(addr, size);
196 kvmppc_set_xics_phys(cpu, addr);
188 if (!icp_native_regs[cpu]) { 197 if (!icp_native_regs[cpu]) {
189 pr_warning("icp_native: Failed ioremap for CPU %d, " 198 pr_warning("icp_native: Failed ioremap for CPU %d, "
190 "interrupt server #0x%x, addr %#lx\n", 199 "interrupt server #0x%x, addr %#lx\n",
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-10 00:46:23 -0500