diff options
Diffstat (limited to 'arch/powerpc/kvm/book3s_hv_rm_xics.c')
-rw-r--r-- | arch/powerpc/kvm/book3s_hv_rm_xics.c | 406 |
1 files changed, 406 insertions, 0 deletions
diff --git a/arch/powerpc/kvm/book3s_hv_rm_xics.c b/arch/powerpc/kvm/book3s_hv_rm_xics.c new file mode 100644 index 000000000000..b4b0082f761c --- /dev/null +++ b/arch/powerpc/kvm/book3s_hv_rm_xics.c | |||
@@ -0,0 +1,406 @@ | |||
1 | /* | ||
2 | * Copyright 2012 Michael Ellerman, IBM Corporation. | ||
3 | * Copyright 2012 Benjamin Herrenschmidt, IBM Corporation | ||
4 | * | ||
5 | * This program is free software; you can redistribute it and/or modify | ||
6 | * it under the terms of the GNU General Public License, version 2, as | ||
7 | * published by the Free Software Foundation. | ||
8 | */ | ||
9 | |||
10 | #include <linux/kernel.h> | ||
11 | #include <linux/kvm_host.h> | ||
12 | #include <linux/err.h> | ||
13 | |||
14 | #include <asm/kvm_book3s.h> | ||
15 | #include <asm/kvm_ppc.h> | ||
16 | #include <asm/hvcall.h> | ||
17 | #include <asm/xics.h> | ||
18 | #include <asm/debug.h> | ||
19 | #include <asm/synch.h> | ||
20 | #include <asm/ppc-opcode.h> | ||
21 | |||
22 | #include "book3s_xics.h" | ||
23 | |||
24 | #define DEBUG_PASSUP | ||
25 | |||
26 | static inline void rm_writeb(unsigned long paddr, u8 val) | ||
27 | { | ||
28 | __asm__ __volatile__("sync; stbcix %0,0,%1" | ||
29 | : : "r" (val), "r" (paddr) : "memory"); | ||
30 | } | ||
31 | |||
32 | static void icp_rm_set_vcpu_irq(struct kvm_vcpu *vcpu, | ||
33 | struct kvm_vcpu *this_vcpu) | ||
34 | { | ||
35 | struct kvmppc_icp *this_icp = this_vcpu->arch.icp; | ||
36 | unsigned long xics_phys; | ||
37 | int cpu; | ||
38 | |||
39 | /* Mark the target VCPU as having an interrupt pending */ | ||
40 | vcpu->stat.queue_intr++; | ||
41 | set_bit(BOOK3S_IRQPRIO_EXTERNAL_LEVEL, &vcpu->arch.pending_exceptions); | ||
42 | |||
43 | /* Kick self ? Just set MER and return */ | ||
44 | if (vcpu == this_vcpu) { | ||
45 | mtspr(SPRN_LPCR, mfspr(SPRN_LPCR) | LPCR_MER); | ||
46 | return; | ||
47 | } | ||
48 | |||
49 | /* Check if the core is loaded, if not, too hard */ | ||
50 | cpu = vcpu->cpu; | ||
51 | if (cpu < 0 || cpu >= nr_cpu_ids) { | ||
52 | this_icp->rm_action |= XICS_RM_KICK_VCPU; | ||
53 | this_icp->rm_kick_target = vcpu; | ||
54 | return; | ||
55 | } | ||
56 | /* In SMT cpu will always point to thread 0, we adjust it */ | ||
57 | cpu += vcpu->arch.ptid; | ||
58 | |||
59 | /* Not too hard, then poke the target */ | ||
60 | xics_phys = paca[cpu].kvm_hstate.xics_phys; | ||
61 | rm_writeb(xics_phys + XICS_MFRR, IPI_PRIORITY); | ||
62 | } | ||
63 | |||
64 | static void icp_rm_clr_vcpu_irq(struct kvm_vcpu *vcpu) | ||
65 | { | ||
66 | /* Note: Only called on self ! */ | ||
67 | clear_bit(BOOK3S_IRQPRIO_EXTERNAL_LEVEL, | ||
68 | &vcpu->arch.pending_exceptions); | ||
69 | mtspr(SPRN_LPCR, mfspr(SPRN_LPCR) & ~LPCR_MER); | ||
70 | } | ||
71 | |||
72 | static inline bool icp_rm_try_update(struct kvmppc_icp *icp, | ||
73 | union kvmppc_icp_state old, | ||
74 | union kvmppc_icp_state new) | ||
75 | { | ||
76 | struct kvm_vcpu *this_vcpu = local_paca->kvm_hstate.kvm_vcpu; | ||
77 | bool success; | ||
78 | |||
79 | /* Calculate new output value */ | ||
80 | new.out_ee = (new.xisr && (new.pending_pri < new.cppr)); | ||
81 | |||
82 | /* Attempt atomic update */ | ||
83 | success = cmpxchg64(&icp->state.raw, old.raw, new.raw) == old.raw; | ||
84 | if (!success) | ||
85 | goto bail; | ||
86 | |||
87 | /* | ||
88 | * Check for output state update | ||
89 | * | ||
90 | * Note that this is racy since another processor could be updating | ||
91 | * the state already. This is why we never clear the interrupt output | ||
92 | * here, we only ever set it. The clear only happens prior to doing | ||
93 | * an update and only by the processor itself. Currently we do it | ||
94 | * in Accept (H_XIRR) and Up_Cppr (H_XPPR). | ||
95 | * | ||
96 | * We also do not try to figure out whether the EE state has changed, | ||
97 | * we unconditionally set it if the new state calls for it. The reason | ||
98 | * for that is that we opportunistically remove the pending interrupt | ||
99 | * flag when raising CPPR, so we need to set it back here if an | ||
100 | * interrupt is still pending. | ||
101 | */ | ||
102 | if (new.out_ee) | ||
103 | icp_rm_set_vcpu_irq(icp->vcpu, this_vcpu); | ||
104 | |||
105 | /* Expose the state change for debug purposes */ | ||
106 | this_vcpu->arch.icp->rm_dbgstate = new; | ||
107 | this_vcpu->arch.icp->rm_dbgtgt = icp->vcpu; | ||
108 | |||
109 | bail: | ||
110 | return success; | ||
111 | } | ||
112 | |||
113 | static inline int check_too_hard(struct kvmppc_xics *xics, | ||
114 | struct kvmppc_icp *icp) | ||
115 | { | ||
116 | return (xics->real_mode_dbg || icp->rm_action) ? H_TOO_HARD : H_SUCCESS; | ||
117 | } | ||
118 | |||
119 | static void icp_rm_down_cppr(struct kvmppc_xics *xics, struct kvmppc_icp *icp, | ||
120 | u8 new_cppr) | ||
121 | { | ||
122 | union kvmppc_icp_state old_state, new_state; | ||
123 | bool resend; | ||
124 | |||
125 | /* | ||
126 | * This handles several related states in one operation: | ||
127 | * | ||
128 | * ICP State: Down_CPPR | ||
129 | * | ||
130 | * Load CPPR with new value and if the XISR is 0 | ||
131 | * then check for resends: | ||
132 | * | ||
133 | * ICP State: Resend | ||
134 | * | ||
135 | * If MFRR is more favored than CPPR, check for IPIs | ||
136 | * and notify ICS of a potential resend. This is done | ||
137 | * asynchronously (when used in real mode, we will have | ||
138 | * to exit here). | ||
139 | * | ||
140 | * We do not handle the complete Check_IPI as documented | ||
141 | * here. In the PAPR, this state will be used for both | ||
142 | * Set_MFRR and Down_CPPR. However, we know that we aren't | ||
143 | * changing the MFRR state here so we don't need to handle | ||
144 | * the case of an MFRR causing a reject of a pending irq, | ||
145 | * this will have been handled when the MFRR was set in the | ||
146 | * first place. | ||
147 | * | ||
148 | * Thus we don't have to handle rejects, only resends. | ||
149 | * | ||
150 | * When implementing real mode for HV KVM, resend will lead to | ||
151 | * a H_TOO_HARD return and the whole transaction will be handled | ||
152 | * in virtual mode. | ||
153 | */ | ||
154 | do { | ||
155 | old_state = new_state = ACCESS_ONCE(icp->state); | ||
156 | |||
157 | /* Down_CPPR */ | ||
158 | new_state.cppr = new_cppr; | ||
159 | |||
160 | /* | ||
161 | * Cut down Resend / Check_IPI / IPI | ||
162 | * | ||
163 | * The logic is that we cannot have a pending interrupt | ||
164 | * trumped by an IPI at this point (see above), so we | ||
165 | * know that either the pending interrupt is already an | ||
166 | * IPI (in which case we don't care to override it) or | ||
167 | * it's either more favored than us or non existent | ||
168 | */ | ||
169 | if (new_state.mfrr < new_cppr && | ||
170 | new_state.mfrr <= new_state.pending_pri) { | ||
171 | new_state.pending_pri = new_state.mfrr; | ||
172 | new_state.xisr = XICS_IPI; | ||
173 | } | ||
174 | |||
175 | /* Latch/clear resend bit */ | ||
176 | resend = new_state.need_resend; | ||
177 | new_state.need_resend = 0; | ||
178 | |||
179 | } while (!icp_rm_try_update(icp, old_state, new_state)); | ||
180 | |||
181 | /* | ||
182 | * Now handle resend checks. Those are asynchronous to the ICP | ||
183 | * state update in HW (ie bus transactions) so we can handle them | ||
184 | * separately here as well. | ||
185 | */ | ||
186 | if (resend) | ||
187 | icp->rm_action |= XICS_RM_CHECK_RESEND; | ||
188 | } | ||
189 | |||
190 | |||
191 | unsigned long kvmppc_rm_h_xirr(struct kvm_vcpu *vcpu) | ||
192 | { | ||
193 | union kvmppc_icp_state old_state, new_state; | ||
194 | struct kvmppc_xics *xics = vcpu->kvm->arch.xics; | ||
195 | struct kvmppc_icp *icp = vcpu->arch.icp; | ||
196 | u32 xirr; | ||
197 | |||
198 | if (!xics || !xics->real_mode) | ||
199 | return H_TOO_HARD; | ||
200 | |||
201 | /* First clear the interrupt */ | ||
202 | icp_rm_clr_vcpu_irq(icp->vcpu); | ||
203 | |||
204 | /* | ||
205 | * ICP State: Accept_Interrupt | ||
206 | * | ||
207 | * Return the pending interrupt (if any) along with the | ||
208 | * current CPPR, then clear the XISR & set CPPR to the | ||
209 | * pending priority | ||
210 | */ | ||
211 | do { | ||
212 | old_state = new_state = ACCESS_ONCE(icp->state); | ||
213 | |||
214 | xirr = old_state.xisr | (((u32)old_state.cppr) << 24); | ||
215 | if (!old_state.xisr) | ||
216 | break; | ||
217 | new_state.cppr = new_state.pending_pri; | ||
218 | new_state.pending_pri = 0xff; | ||
219 | new_state.xisr = 0; | ||
220 | |||
221 | } while (!icp_rm_try_update(icp, old_state, new_state)); | ||
222 | |||
223 | /* Return the result in GPR4 */ | ||
224 | vcpu->arch.gpr[4] = xirr; | ||
225 | |||
226 | return check_too_hard(xics, icp); | ||
227 | } | ||
228 | |||
229 | int kvmppc_rm_h_ipi(struct kvm_vcpu *vcpu, unsigned long server, | ||
230 | unsigned long mfrr) | ||
231 | { | ||
232 | union kvmppc_icp_state old_state, new_state; | ||
233 | struct kvmppc_xics *xics = vcpu->kvm->arch.xics; | ||
234 | struct kvmppc_icp *icp, *this_icp = vcpu->arch.icp; | ||
235 | u32 reject; | ||
236 | bool resend; | ||
237 | bool local; | ||
238 | |||
239 | if (!xics || !xics->real_mode) | ||
240 | return H_TOO_HARD; | ||
241 | |||
242 | local = this_icp->server_num == server; | ||
243 | if (local) | ||
244 | icp = this_icp; | ||
245 | else | ||
246 | icp = kvmppc_xics_find_server(vcpu->kvm, server); | ||
247 | if (!icp) | ||
248 | return H_PARAMETER; | ||
249 | |||
250 | /* | ||
251 | * ICP state: Set_MFRR | ||
252 | * | ||
253 | * If the CPPR is more favored than the new MFRR, then | ||
254 | * nothing needs to be done as there can be no XISR to | ||
255 | * reject. | ||
256 | * | ||
257 | * If the CPPR is less favored, then we might be replacing | ||
258 | * an interrupt, and thus need to possibly reject it as in | ||
259 | * | ||
260 | * ICP state: Check_IPI | ||
261 | */ | ||
262 | do { | ||
263 | old_state = new_state = ACCESS_ONCE(icp->state); | ||
264 | |||
265 | /* Set_MFRR */ | ||
266 | new_state.mfrr = mfrr; | ||
267 | |||
268 | /* Check_IPI */ | ||
269 | reject = 0; | ||
270 | resend = false; | ||
271 | if (mfrr < new_state.cppr) { | ||
272 | /* Reject a pending interrupt if not an IPI */ | ||
273 | if (mfrr <= new_state.pending_pri) | ||
274 | reject = new_state.xisr; | ||
275 | new_state.pending_pri = mfrr; | ||
276 | new_state.xisr = XICS_IPI; | ||
277 | } | ||
278 | |||
279 | if (mfrr > old_state.mfrr && mfrr > new_state.cppr) { | ||
280 | resend = new_state.need_resend; | ||
281 | new_state.need_resend = 0; | ||
282 | } | ||
283 | } while (!icp_rm_try_update(icp, old_state, new_state)); | ||
284 | |||
285 | /* Pass rejects to virtual mode */ | ||
286 | if (reject && reject != XICS_IPI) { | ||
287 | this_icp->rm_action |= XICS_RM_REJECT; | ||
288 | this_icp->rm_reject = reject; | ||
289 | } | ||
290 | |||
291 | /* Pass resends to virtual mode */ | ||
292 | if (resend) | ||
293 | this_icp->rm_action |= XICS_RM_CHECK_RESEND; | ||
294 | |||
295 | return check_too_hard(xics, this_icp); | ||
296 | } | ||
297 | |||
298 | int kvmppc_rm_h_cppr(struct kvm_vcpu *vcpu, unsigned long cppr) | ||
299 | { | ||
300 | union kvmppc_icp_state old_state, new_state; | ||
301 | struct kvmppc_xics *xics = vcpu->kvm->arch.xics; | ||
302 | struct kvmppc_icp *icp = vcpu->arch.icp; | ||
303 | u32 reject; | ||
304 | |||
305 | if (!xics || !xics->real_mode) | ||
306 | return H_TOO_HARD; | ||
307 | |||
308 | /* | ||
309 | * ICP State: Set_CPPR | ||
310 | * | ||
311 | * We can safely compare the new value with the current | ||
312 | * value outside of the transaction as the CPPR is only | ||
313 | * ever changed by the processor on itself | ||
314 | */ | ||
315 | if (cppr > icp->state.cppr) { | ||
316 | icp_rm_down_cppr(xics, icp, cppr); | ||
317 | goto bail; | ||
318 | } else if (cppr == icp->state.cppr) | ||
319 | return H_SUCCESS; | ||
320 | |||
321 | /* | ||
322 | * ICP State: Up_CPPR | ||
323 | * | ||
324 | * The processor is raising its priority, this can result | ||
325 | * in a rejection of a pending interrupt: | ||
326 | * | ||
327 | * ICP State: Reject_Current | ||
328 | * | ||
329 | * We can remove EE from the current processor, the update | ||
330 | * transaction will set it again if needed | ||
331 | */ | ||
332 | icp_rm_clr_vcpu_irq(icp->vcpu); | ||
333 | |||
334 | do { | ||
335 | old_state = new_state = ACCESS_ONCE(icp->state); | ||
336 | |||
337 | reject = 0; | ||
338 | new_state.cppr = cppr; | ||
339 | |||
340 | if (cppr <= new_state.pending_pri) { | ||
341 | reject = new_state.xisr; | ||
342 | new_state.xisr = 0; | ||
343 | new_state.pending_pri = 0xff; | ||
344 | } | ||
345 | |||
346 | } while (!icp_rm_try_update(icp, old_state, new_state)); | ||
347 | |||
348 | /* Pass rejects to virtual mode */ | ||
349 | if (reject && reject != XICS_IPI) { | ||
350 | icp->rm_action |= XICS_RM_REJECT; | ||
351 | icp->rm_reject = reject; | ||
352 | } | ||
353 | bail: | ||
354 | return check_too_hard(xics, icp); | ||
355 | } | ||
356 | |||
357 | int kvmppc_rm_h_eoi(struct kvm_vcpu *vcpu, unsigned long xirr) | ||
358 | { | ||
359 | struct kvmppc_xics *xics = vcpu->kvm->arch.xics; | ||
360 | struct kvmppc_icp *icp = vcpu->arch.icp; | ||
361 | struct kvmppc_ics *ics; | ||
362 | struct ics_irq_state *state; | ||
363 | u32 irq = xirr & 0x00ffffff; | ||
364 | u16 src; | ||
365 | |||
366 | if (!xics || !xics->real_mode) | ||
367 | return H_TOO_HARD; | ||
368 | |||
369 | /* | ||
370 | * ICP State: EOI | ||
371 | * | ||
372 | * Note: If EOI is incorrectly used by SW to lower the CPPR | ||
373 | * value (ie more favored), we do not check for rejection of | ||
374 | * a pending interrupt, this is a SW error and PAPR sepcifies | ||
375 | * that we don't have to deal with it. | ||
376 | * | ||
377 | * The sending of an EOI to the ICS is handled after the | ||
378 | * CPPR update | ||
379 | * | ||
380 | * ICP State: Down_CPPR which we handle | ||
381 | * in a separate function as it's shared with H_CPPR. | ||
382 | */ | ||
383 | icp_rm_down_cppr(xics, icp, xirr >> 24); | ||
384 | |||
385 | /* IPIs have no EOI */ | ||
386 | if (irq == XICS_IPI) | ||
387 | goto bail; | ||
388 | /* | ||
389 | * EOI handling: If the interrupt is still asserted, we need to | ||
390 | * resend it. We can take a lockless "peek" at the ICS state here. | ||
391 | * | ||
392 | * "Message" interrupts will never have "asserted" set | ||
393 | */ | ||
394 | ics = kvmppc_xics_find_ics(xics, irq, &src); | ||
395 | if (!ics) | ||
396 | goto bail; | ||
397 | state = &ics->irq_state[src]; | ||
398 | |||
399 | /* Still asserted, resend it, we make it look like a reject */ | ||
400 | if (state->asserted) { | ||
401 | icp->rm_action |= XICS_RM_REJECT; | ||
402 | icp->rm_reject = irq; | ||
403 | } | ||
404 | bail: | ||
405 | return check_too_hard(xics, icp); | ||
406 | } | ||