KVM: PPC: Book3S HV: Make secondary threads more robust against stray IPIs

Currently on POWER7, if we are running the guest on a core and we don't
need all the hardware threads, we do nothing to ensure that the unused
threads aren't executing in the kernel (other than checking that they
are offline).  We just assume they're napping and we don't do anything
to stop them trying to enter the kernel while the guest is running.
This means that a stray IPI can wake up the hardware thread and it will
then try to enter the kernel, but since the core is in guest context,
it will execute code from the guest in hypervisor mode once it turns the
MMU on, which tends to lead to crashes or hangs in the host.

This fixes the problem by adding two new one-byte flags in the
kvmppc_host_state structure in the PACA which are used to interlock
between the primary thread and the unused secondary threads when entering
the guest.  With these flags, the primary thread can ensure that the
unused secondaries are not already in kernel mode (i.e. handling a stray
IPI) and then indicate that they should not try to enter the kernel
if they do get woken for any reason.  Instead they will go into KVM code,
find that there is no vcpu to run, acknowledge and clear the IPI and go
back to nap mode.

Signed-off-by: Paul Mackerras <paulus@samba.org>
Signed-off-by: Alexander Graf <agraf@suse.de>
Signed-off-by: Avi Kivity <avi@redhat.com>

3 files changed, 16 insertions, 5 deletions

diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index bbede5882c5b..2abcf7d4b29c 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -540,6 +540,8 @@ int main(void)
         HSTATE_FIELD(HSTATE_IN_GUEST, in_guest);
         HSTATE_FIELD(HSTATE_RESTORE_HID5, restore_hid5);
         HSTATE_FIELD(HSTATE_NAPPING, napping);
+        HSTATE_FIELD(HSTATE_HWTHREAD_REQ, hwthread_req);
+        HSTATE_FIELD(HSTATE_HWTHREAD_STATE, hwthread_state);
 
 #ifdef CONFIG_KVM_BOOK3S_64_HV
         HSTATE_FIELD(HSTATE_KVM_VCPU, kvm_vcpu);
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index cb705fdbb458..8829b1095f7f 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -63,11 +63,13 @@ BEGIN_FTR_SECTION
         GET_PACA(r13)
 
 #ifdef CONFIG_KVM_BOOK3S_64_HV
-        lbz     r0,PACAPROCSTART(r13)
-        cmpwi   r0,0x80
-        bne     1f
-        li      r0,1
-        stb     r0,PACAPROCSTART(r13)
+        li      r0,KVM_HWTHREAD_IN_KERNEL
+        stb     r0,HSTATE_HWTHREAD_STATE(r13)
+        /* Order setting hwthread_state vs. testing hwthread_req */
+        sync
+        lbz     r0,HSTATE_HWTHREAD_REQ(r13)
+        cmpwi   r0,0
+        beq     1f
         b       kvm_start_guest
 1:
 #endif
diff --git a/arch/powerpc/kernel/idle_power7.S b/arch/powerpc/kernel/idle_power7.S
index 0cdc9a392839..7140d838339e 100644
--- a/arch/powerpc/kernel/idle_power7.S
+++ b/arch/powerpc/kernel/idle_power7.S
@@ -16,6 +16,7 @@
 #include <asm/asm-offsets.h>
 #include <asm/ppc-opcode.h>
 #include <asm/hw_irq.h>
+#include <asm/kvm_book3s_asm.h>
 
 #undef DEBUG
 
@@ -81,6 +82,12 @@ _GLOBAL(power7_idle)
         std     r9,_MSR(r1)
         std     r1,PACAR1(r13)
 
+#ifdef CONFIG_KVM_BOOK3S_64_HV
+        /* Tell KVM we're napping */
+        li      r4,KVM_HWTHREAD_IN_NAP
+        stb     r4,HSTATE_HWTHREAD_STATE(r13)
+#endif
+
         /* Magic NAP mode enter sequence */
         std     r0,0(r1)
         ptesync