KVM: PPC: Implement H_CEDE hcall for book3s_hv in real-mode code

With a KVM guest operating in SMT4 mode (i.e. 4 hardware threads per
core), whenever a CPU goes idle, we have to pull all the other
hardware threads in the core out of the guest, because the H_CEDE
hcall is handled in the kernel.  This is inefficient.

This adds code to book3s_hv_rmhandlers.S to handle the H_CEDE hcall
in real mode.  When a guest vcpu does an H_CEDE hcall, we now only
exit to the kernel if all the other vcpus in the same core are also
idle.  Otherwise we mark this vcpu as napping, save state that could
be lost in nap mode (mainly GPRs and FPRs), and execute the nap
instruction.  When the thread wakes up, because of a decrementer or
external interrupt, we come back in at kvm_start_guest (from the
system reset interrupt vector), find the `napping' flag set in the
paca, and go to the resume path.

This has some other ramifications.  First, when starting a core, we
now start all the threads, both those that are immediately runnable and
those that are idle.  This is so that we don't have to pull all the
threads out of the guest when an idle thread gets a decrementer interrupt
and wants to start running.  In fact the idle threads will all start
with the H_CEDE hcall returning; being idle they will just do another
H_CEDE immediately and go to nap mode.

This required some changes to kvmppc_run_core() and kvmppc_run_vcpu().
These functions have been restructured to make them simpler and clearer.
We introduce a level of indirection in the wait queue that gets woken
when external and decrementer interrupts get generated for a vcpu, so
that we can have the 4 vcpus in a vcore using the same wait queue.
We need this because the 4 vcpus are being handled by one thread.

Secondly, when we need to exit from the guest to the kernel, we now
have to generate an IPI for any napping threads, because an HDEC
interrupt doesn't wake up a napping thread.

Thirdly, we now need to be able to handle virtual external interrupts
and decrementer interrupts becoming pending while a thread is napping,
and deliver those interrupts to the guest when the thread wakes.
This is done in kvmppc_cede_reentry, just before fast_guest_return.

Finally, since we are not using the generic kvm_vcpu_block for book3s_hv,
and hence not calling kvm_arch_vcpu_runnable, we can remove the #ifdef
from kvm_arch_vcpu_runnable.

Signed-off-by: Paul Mackerras <paulus@samba.org>
Signed-off-by: Alexander Graf <agraf@suse.de>

1 files changed, 263 insertions, 34 deletions

diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
index bc6ade933089..f422231d9235 100644
--- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S
+++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
@@ -52,7 +52,7 @@ kvmppc_skip_Hinterrupt:
         b       .
 
 /*
- * Call kvmppc_handler_trampoline_enter in real mode.
+ * Call kvmppc_hv_entry in real mode.
  * Must be called with interrupts hard-disabled.
  *
  * Input Registers:
@@ -92,6 +92,12 @@ _GLOBAL(kvmppc_hv_entry_trampoline)
 kvm_start_guest:
         ld      r1,PACAEMERGSP(r13)
         subi    r1,r1,STACK_FRAME_OVERHEAD
+        ld      r2,PACATOC(r13)
+
+        /* were we napping due to cede? */
+        lbz     r0,HSTATE_NAPPING(r13)
+        cmpwi   r0,0
+        bne     kvm_end_cede
 
         /* get vcpu pointer */
         ld      r4, HSTATE_KVM_VCPU(r13)
@@ -279,15 +285,9 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
         cmpwi   r0,0
         beq     20b
 
-        /* Set LPCR.  Set the MER bit if there is a pending external irq. */
+        /* Set LPCR and RMOR. */
 10:     ld      r8,KVM_LPCR(r9)
-        ld      r0,VCPU_PENDING_EXC(r4)
-        li      r7,(1 << BOOK3S_IRQPRIO_EXTERNAL)
-        oris    r7,r7,(1 << BOOK3S_IRQPRIO_EXTERNAL_LEVEL)@h
-        and.    r0,r0,r7
-        beq     11f
-        ori     r8,r8,LPCR_MER
-11:     mtspr   SPRN_LPCR,r8
+        mtspr   SPRN_LPCR,r8
         ld      r8,KVM_RMOR(r9)
         mtspr   SPRN_RMOR,r8
         isync
@@ -451,19 +451,50 @@ toc_tlbie_lock:
         mtctr   r6
         mtxer   r7
 
-        /* Move SRR0 and SRR1 into the respective regs */
+kvmppc_cede_reentry:            /* r4 = vcpu, r13 = paca */
         ld      r6, VCPU_SRR0(r4)
         ld      r7, VCPU_SRR1(r4)
-        mtspr   SPRN_SRR0, r6
-        mtspr   SPRN_SRR1, r7
-
         ld      r10, VCPU_PC(r4)
+        ld      r11, VCPU_MSR(r4)       /* r11 = vcpu->arch.msr & ~MSR_HV */
 
-        ld      r11, VCPU_MSR(r4)       /* r10 = vcpu->arch.msr & ~MSR_HV */
         rldicl  r11, r11, 63 - MSR_HV_LG, 1
         rotldi  r11, r11, 1 + MSR_HV_LG
         ori     r11, r11, MSR_ME
 
+        /* Check if we can deliver an external or decrementer interrupt now */
+        ld      r0,VCPU_PENDING_EXC(r4)
+        li      r8,(1 << BOOK3S_IRQPRIO_EXTERNAL)
+        oris    r8,r8,(1 << BOOK3S_IRQPRIO_EXTERNAL_LEVEL)@h
+        and     r0,r0,r8
+        cmpdi   cr1,r0,0
+        andi.   r0,r11,MSR_EE
+        beq     cr1,11f
+BEGIN_FTR_SECTION
+        mfspr   r8,SPRN_LPCR
+        ori     r8,r8,LPCR_MER
+        mtspr   SPRN_LPCR,r8
+        isync
+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
+        beq     5f
+        li      r0,BOOK3S_INTERRUPT_EXTERNAL
+12:     mr      r6,r10
+        mr      r10,r0
+        mr      r7,r11
+        li      r11,(MSR_ME << 1) | 1   /* synthesize MSR_SF | MSR_ME */
+        rotldi  r11,r11,63
+        b       5f
+11:     beq     5f
+        mfspr   r0,SPRN_DEC
+        cmpwi   r0,0
+        li      r0,BOOK3S_INTERRUPT_DECREMENTER
+        blt     12b
+
+        /* Move SRR0 and SRR1 into the respective regs */
+5:      mtspr   SPRN_SRR0, r6
+        mtspr   SPRN_SRR1, r7
+        li      r0,0
+        stb     r0,VCPU_CEDED(r4)       /* cancel cede */
+
 fast_guest_return:
         mtspr   SPRN_HSRR0,r10
         mtspr   SPRN_HSRR1,r11
@@ -577,21 +608,20 @@ kvmppc_interrupt:
         /* See if this is something we can handle in real mode */
         cmpwi   r12,BOOK3S_INTERRUPT_SYSCALL
         beq     hcall_try_real_mode
-hcall_real_cont:
 
         /* Check for mediated interrupts (could be done earlier really ...) */
 BEGIN_FTR_SECTION
         cmpwi   r12,BOOK3S_INTERRUPT_EXTERNAL
         bne+    1f
-        ld      r5,VCPU_KVM(r9)
-        ld      r5,KVM_LPCR(r5)
         andi.   r0,r11,MSR_EE
         beq     1f
+        mfspr   r5,SPRN_LPCR
         andi.   r0,r5,LPCR_MER
         bne     bounce_ext_interrupt
 1:
 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
 
+hcall_real_cont:                /* r9 = vcpu, r12 = trap, r13 = paca */
         /* Save DEC */
         mfspr   r5,SPRN_DEC
         mftb    r6
@@ -685,7 +715,7 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_201)
         slbia
         ptesync
 
-hdec_soon:
+hdec_soon:                      /* r9 = vcpu, r12 = trap, r13 = paca */
 BEGIN_FTR_SECTION
         b       32f
 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
@@ -703,6 +733,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
         addi    r0,r3,0x100
         stwcx.  r0,0,r6
         bne     41b
+        lwsync
 
         /*
          * At this point we have an interrupt that we have to pass
@@ -716,18 +747,39 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
          * interrupt, since the other threads will already be on their
          * way here in that case.
          */
+        cmpwi   r3,0x100        /* Are we the first here? */
+        bge     43f
+        cmpwi   r3,1            /* Are any other threads in the guest? */
+        ble     43f
         cmpwi   r12,BOOK3S_INTERRUPT_HV_DECREMENTER
         beq     40f
-        cmpwi   r3,0x100        /* Are we the first here? */
-        bge     40f
-        cmpwi   r3,1
-        ble     40f
         li      r0,0
         mtspr   SPRN_HDEC,r0
 40:
+        /*
+         * Send an IPI to any napping threads, since an HDEC interrupt
+         * doesn't wake CPUs up from nap.
+         */
+        lwz     r3,VCORE_NAPPING_THREADS(r5)
+        lwz     r4,VCPU_PTID(r9)
+        li      r0,1
+        sldi    r0,r0,r4
+        andc.   r3,r3,r0                /* no sense IPI'ing ourselves */
+        beq     43f
+        mulli   r4,r4,PACA_SIZE         /* get paca for thread 0 */
+        subf    r6,r4,r13
+42:     andi.   r0,r3,1
+        beq     44f
+        ld      r8,HSTATE_XICS_PHYS(r6) /* get thread's XICS reg addr */
+        li      r0,IPI_PRIORITY
+        li      r7,XICS_QIRR
+        stbcix  r0,r7,r8                /* trigger the IPI */
+44:     srdi.   r3,r3,1
+        addi    r6,r6,PACA_SIZE
+        bne     42b
 
         /* Secondary threads wait for primary to do partition switch */
-        ld      r4,VCPU_KVM(r9)         /* pointer to struct kvm */
+43:     ld      r4,VCPU_KVM(r9)         /* pointer to struct kvm */
         ld      r5,HSTATE_KVM_VCORE(r13)
         lwz     r3,VCPU_PTID(r9)
         cmpwi   r3,0
@@ -1080,7 +1132,6 @@ hcall_try_real_mode:
 hcall_real_fallback:
         li      r12,BOOK3S_INTERRUPT_SYSCALL
         ld      r9, HSTATE_KVM_VCPU(r13)
-        ld      r11, VCPU_MSR(r9)
 
         b       hcall_real_cont
 
@@ -1142,7 +1193,7 @@ hcall_real_table:
         .long   0               /* 0xd4 */
         .long   0               /* 0xd8 */
         .long   0               /* 0xdc */
-        .long   0               /* 0xe0 */
+        .long   .kvmppc_h_cede - hcall_real_table
         .long   0               /* 0xe4 */
         .long   0               /* 0xe8 */
         .long   0               /* 0xec */
@@ -1171,7 +1222,8 @@ bounce_ext_interrupt:
         mtspr   SPRN_SRR0,r10
         mtspr   SPRN_SRR1,r11
         li      r10,BOOK3S_INTERRUPT_EXTERNAL
-        LOAD_REG_IMMEDIATE(r11,MSR_SF | MSR_ME);
+        li      r11,(MSR_ME << 1) | 1   /* synthesize MSR_SF | MSR_ME */
+        rotldi  r11,r11,63
         b       fast_guest_return
 
 _GLOBAL(kvmppc_h_set_dabr)
@@ -1180,6 +1232,178 @@ _GLOBAL(kvmppc_h_set_dabr)
         li      r3,0
         blr
 
+_GLOBAL(kvmppc_h_cede)
+        ori     r11,r11,MSR_EE
+        std     r11,VCPU_MSR(r3)
+        li      r0,1
+        stb     r0,VCPU_CEDED(r3)
+        sync                    /* order setting ceded vs. testing prodded */
+        lbz     r5,VCPU_PRODDED(r3)
+        cmpwi   r5,0
+        bne     1f
+        li      r0,0            /* set trap to 0 to say hcall is handled */
+        stw     r0,VCPU_TRAP(r3)
+        li      r0,H_SUCCESS
+        std     r0,VCPU_GPR(r3)(r3)
+BEGIN_FTR_SECTION
+        b       2f              /* just send it up to host on 970 */
+END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_206)
+
+        /*
+         * Set our bit in the bitmask of napping threads unless all the
+         * other threads are already napping, in which case we send this
+         * up to the host.
+         */
+        ld      r5,HSTATE_KVM_VCORE(r13)
+        lwz     r6,VCPU_PTID(r3)
+        lwz     r8,VCORE_ENTRY_EXIT(r5)
+        clrldi  r8,r8,56
+        li      r0,1
+        sld     r0,r0,r6
+        addi    r6,r5,VCORE_NAPPING_THREADS
+31:     lwarx   r4,0,r6
+        or      r4,r4,r0
+        popcntw r7,r4
+        cmpw    r7,r8
+        bge     2f
+        stwcx.  r4,0,r6
+        bne     31b
+        li      r0,1
+        stb     r0,HSTATE_NAPPING(r13)
+        /* order napping_threads update vs testing entry_exit_count */
+        lwsync
+        mr      r4,r3
+        lwz     r7,VCORE_ENTRY_EXIT(r5)
+        cmpwi   r7,0x100
+        bge     33f             /* another thread already exiting */
+
+/*
+ * Although not specifically required by the architecture, POWER7
+ * preserves the following registers in nap mode, even if an SMT mode
+ * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3,
+ * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR.
+ */
+        /* Save non-volatile GPRs */
+        std     r14, VCPU_GPR(r14)(r3)
+        std     r15, VCPU_GPR(r15)(r3)
+        std     r16, VCPU_GPR(r16)(r3)
+        std     r17, VCPU_GPR(r17)(r3)
+        std     r18, VCPU_GPR(r18)(r3)
+        std     r19, VCPU_GPR(r19)(r3)
+        std     r20, VCPU_GPR(r20)(r3)
+        std     r21, VCPU_GPR(r21)(r3)
+        std     r22, VCPU_GPR(r22)(r3)
+        std     r23, VCPU_GPR(r23)(r3)
+        std     r24, VCPU_GPR(r24)(r3)
+        std     r25, VCPU_GPR(r25)(r3)
+        std     r26, VCPU_GPR(r26)(r3)
+        std     r27, VCPU_GPR(r27)(r3)
+        std     r28, VCPU_GPR(r28)(r3)
+        std     r29, VCPU_GPR(r29)(r3)
+        std     r30, VCPU_GPR(r30)(r3)
+        std     r31, VCPU_GPR(r31)(r3)
+
+        /* save FP state */
+        bl      .kvmppc_save_fp
+
+        /*
+         * Take a nap until a decrementer or external interrupt occurs,
+         * with PECE1 (wake on decr) and PECE0 (wake on external) set in LPCR
+         */
+        li      r0,0x80
+        stb     r0,PACAPROCSTART(r13)
+        mfspr   r5,SPRN_LPCR
+        ori     r5,r5,LPCR_PECE0 | LPCR_PECE1
+        mtspr   SPRN_LPCR,r5
+        isync
+        li      r0, 0
+        std     r0, HSTATE_SCRATCH0(r13)
+        ptesync
+        ld      r0, HSTATE_SCRATCH0(r13)
+1:      cmpd    r0, r0
+        bne     1b
+        nap
+        b       .
+
+kvm_end_cede:
+        /* Woken by external or decrementer interrupt */
+        ld      r1, HSTATE_HOST_R1(r13)
+        ld      r2, PACATOC(r13)
+
+        /* If we're a secondary thread and we got here by an IPI, ack it */
+        ld      r4,HSTATE_KVM_VCPU(r13)
+        lwz     r3,VCPU_PTID(r4)
+        cmpwi   r3,0
+        beq     27f
+        mfspr   r3,SPRN_SRR1
+        rlwinm  r3,r3,44-31,0x7         /* extract wake reason field */
+        cmpwi   r3,4                    /* was it an external interrupt? */
+        bne     27f
+        ld      r5, HSTATE_XICS_PHYS(r13)
+        li      r0,0xff
+        li      r6,XICS_QIRR
+        li      r7,XICS_XIRR
+        lwzcix  r8,r5,r7                /* ack the interrupt */
+        sync
+        stbcix  r0,r5,r6                /* clear it */
+        stwcix  r8,r5,r7                /* EOI it */
+27:
+        /* load up FP state */
+        bl      kvmppc_load_fp
+
+        /* Load NV GPRS */
+        ld      r14, VCPU_GPR(r14)(r4)
+        ld      r15, VCPU_GPR(r15)(r4)
+        ld      r16, VCPU_GPR(r16)(r4)
+        ld      r17, VCPU_GPR(r17)(r4)
+        ld      r18, VCPU_GPR(r18)(r4)
+        ld      r19, VCPU_GPR(r19)(r4)
+        ld      r20, VCPU_GPR(r20)(r4)
+        ld      r21, VCPU_GPR(r21)(r4)
+        ld      r22, VCPU_GPR(r22)(r4)
+        ld      r23, VCPU_GPR(r23)(r4)
+        ld      r24, VCPU_GPR(r24)(r4)
+        ld      r25, VCPU_GPR(r25)(r4)
+        ld      r26, VCPU_GPR(r26)(r4)
+        ld      r27, VCPU_GPR(r27)(r4)
+        ld      r28, VCPU_GPR(r28)(r4)
+        ld      r29, VCPU_GPR(r29)(r4)
+        ld      r30, VCPU_GPR(r30)(r4)
+        ld      r31, VCPU_GPR(r31)(r4)
+
+        /* clear our bit in vcore->napping_threads */
+33:     ld      r5,HSTATE_KVM_VCORE(r13)
+        lwz     r3,VCPU_PTID(r4)
+        li      r0,1
+        sld     r0,r0,r3
+        addi    r6,r5,VCORE_NAPPING_THREADS
+32:     lwarx   r7,0,r6
+        andc    r7,r7,r0
+        stwcx.  r7,0,r6
+        bne     32b
+        li      r0,0
+        stb     r0,HSTATE_NAPPING(r13)
+
+        /* see if any other thread is already exiting */
+        lwz     r0,VCORE_ENTRY_EXIT(r5)
+        cmpwi   r0,0x100
+        blt     kvmppc_cede_reentry     /* if not go back to guest */
+
+        /* some threads are exiting, so go to the guest exit path */
+        b       hcall_real_fallback
+
+        /* cede when already previously prodded case */
+1:      li      r0,0
+        stb     r0,VCPU_PRODDED(r3)
+        sync                    /* order testing prodded vs. clearing ceded */
+        stb     r0,VCPU_CEDED(r3)
+        li      r3,H_SUCCESS
+        blr
+
+        /* we've ceded but we want to give control to the host */
+2:      li      r3,H_TOO_HARD
+        blr
+
 secondary_too_late:
         ld      r5,HSTATE_KVM_VCORE(r13)
         HMT_LOW
@@ -1197,14 +1421,20 @@ secondary_too_late:
         slbmte  r6,r5
 1:      addi    r11,r11,16
         .endr
-        b       50f
 
 secondary_nap:
-        /* Clear any pending IPI */
-50:     ld      r5, HSTATE_XICS_PHYS(r13)
+        /* Clear any pending IPI - assume we're a secondary thread */
+        ld      r5, HSTATE_XICS_PHYS(r13)
+        li      r7, XICS_XIRR
+        lwzcix  r3, r5, r7              /* ack any pending interrupt */
+        rlwinm. r0, r3, 0, 0xffffff     /* any pending? */
+        beq     37f
+        sync
         li      r0, 0xff
         li      r6, XICS_QIRR
-        stbcix  r0, r5, r6
+        stbcix  r0, r5, r6              /* clear the IPI */
+        stwcix  r3, r5, r7              /* EOI it */
+37:     sync
 
         /* increment the nap count and then go to nap mode */
         ld      r4, HSTATE_KVM_VCORE(r13)
@@ -1214,13 +1444,12 @@ secondary_nap:
         addi    r3, r3, 1
         stwcx.  r3, 0, r4
         bne     51b
-        isync
 
+        li      r3, LPCR_PECE0
         mfspr   r4, SPRN_LPCR
-        li      r0, LPCR_PECE
-        andc    r4, r4, r0
-        ori     r4, r4, LPCR_PECE0      /* exit nap on interrupt */
+        rlwimi  r4, r3, 0, LPCR_PECE0 | LPCR_PECE1
         mtspr   SPRN_LPCR, r4
+        isync
         li      r0, 0
         std     r0, HSTATE_SCRATCH0(r13)
         ptesync