Merge tag 'powerpc-3.19-2' of git://git.kernel.org/pub/scm/linux/kernel/git/mpe/linux

Pull second batch of powerpc updates from Michael Ellerman:
 "The highlight is the series that reworks the idle management on
  powernv, which allows us to use deeper idle states on those machines.

  There's the fix from Anton for the "BUG at kernel/smpboot.c:134!"
  problem.

  An i2c driver for powernv.  This is acked by Wolfram Sang, and he
  asked that we take it through the powerpc tree.

  A fix for audit from rgb at Red Hat, acked by Paul Moore who is one of
  the audit maintainers.

  A patch from Ben to export the symbol map of our OPAL firmware as a
  sysfs file, so that tools can use it.

  Also some CXL fixes, a couple of powerpc perf fixes, a fix for
  smt-enabled, and the patch to add __force to get_user() so we can use
  bitwise types"

* tag 'powerpc-3.19-2' of git://git.kernel.org/pub/scm/linux/kernel/git/mpe/linux:
  powerpc/powernv: Ignore smt-enabled on Power8 and later
  powerpc/uaccess: Allow get_user() with bitwise types
  powerpc/powernv: Expose OPAL firmware symbol map
  powernv/powerpc: Add winkle support for offline cpus
  powernv/cpuidle: Redesign idle states management
  powerpc/powernv: Enable Offline CPUs to enter deep idle states
  powerpc/powernv: Switch off MMU before entering nap/sleep/rvwinkle mode
  i2c: Driver to expose PowerNV platform i2c busses
  powerpc: add little endian flag to syscall_get_arch()
  power/perf/hv-24x7: Use kmem_cache_free() instead of kfree
  powerpc/perf/hv-24x7: Use per-cpu page buffer
  cxl: Unmap MMIO regions when detaching a context
  cxl: Add timeout to process element commands
  cxl: Change contexts_lock to a mutex to fix sleep while atomic bug
  powerpc: Secondary CPUs must set cpu_callin_map after setting active and online

4 files changed, 343 insertions, 56 deletions

diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index 24d78e1871c9..e624f9646350 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -726,5 +726,16 @@ int main(void)
                                         arch.timing_last_enter.tv32.tbl));
 #endif
 
+#ifdef CONFIG_PPC_POWERNV
+        DEFINE(PACA_CORE_IDLE_STATE_PTR,
+                        offsetof(struct paca_struct, core_idle_state_ptr));
+        DEFINE(PACA_THREAD_IDLE_STATE,
+                        offsetof(struct paca_struct, thread_idle_state));
+        DEFINE(PACA_THREAD_MASK,
+                        offsetof(struct paca_struct, thread_mask));
+        DEFINE(PACA_SUBCORE_SIBLING_MASK,
+                        offsetof(struct paca_struct, subcore_sibling_mask));
+#endif
+
         return 0;
 }
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index db08382e19f1..c2df8150bd7a 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -15,6 +15,7 @@
 #include <asm/hw_irq.h>
 #include <asm/exception-64s.h>
 #include <asm/ptrace.h>
+#include <asm/cpuidle.h>
 
 /*
  * We layout physical memory as follows:
@@ -101,23 +102,34 @@ system_reset_pSeries:
 #ifdef CONFIG_PPC_P7_NAP
 BEGIN_FTR_SECTION
         /* Running native on arch 2.06 or later, check if we are
-         * waking up from nap. We only handle no state loss and
-         * supervisor state loss. We do -not- handle hypervisor
-         * state loss at this time.
+         * waking up from nap/sleep/winkle.
          */
         mfspr   r13,SPRN_SRR1
         rlwinm. r13,r13,47-31,30,31
         beq     9f
 
-        /* waking up from powersave (nap) state */
-        cmpwi   cr1,r13,2
-        /* Total loss of HV state is fatal, we could try to use the
-         * PIR to locate a PACA, then use an emergency stack etc...
-         * OPAL v3 based powernv platforms have new idle states
-         * which fall in this catagory.
+        cmpwi   cr3,r13,2
+
+        /*
+         * Check if last bit of HSPGR0 is set. This indicates whether we are
+         * waking up from winkle.
          */
-        bgt     cr1,8f
         GET_PACA(r13)
+        clrldi  r5,r13,63
+        clrrdi  r13,r13,1
+        cmpwi   cr4,r5,1
+        mtspr   SPRN_HSPRG0,r13
+
+        lbz     r0,PACA_THREAD_IDLE_STATE(r13)
+        cmpwi   cr2,r0,PNV_THREAD_NAP
+        bgt     cr2,8f                          /* Either sleep or Winkle */
+
+        /* Waking up from nap should not cause hypervisor state loss */
+        bgt     cr3,.
+
+        /* Waking up from nap */
+        li      r0,PNV_THREAD_RUNNING
+        stb     r0,PACA_THREAD_IDLE_STATE(r13)  /* Clear thread state */
 
 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
         li      r0,KVM_HWTHREAD_IN_KERNEL
@@ -133,7 +145,7 @@ BEGIN_FTR_SECTION
 
         /* Return SRR1 from power7_nap() */
         mfspr   r3,SPRN_SRR1
-        beq     cr1,2f
+        beq     cr3,2f
         b       power7_wakeup_noloss
 2:      b       power7_wakeup_loss
 
@@ -1382,6 +1394,7 @@ machine_check_handle_early:
         MACHINE_CHECK_HANDLER_WINDUP
         GET_PACA(r13)
         ld      r1,PACAR1(r13)
+        li      r3,PNV_THREAD_NAP
         b       power7_enter_nap_mode
 4:
 #endif
diff --git a/arch/powerpc/kernel/idle_power7.S b/arch/powerpc/kernel/idle_power7.S
index 18c0687e5ab3..05adc8bbdef8 100644
--- a/arch/powerpc/kernel/idle_power7.S
+++ b/arch/powerpc/kernel/idle_power7.S
@@ -18,9 +18,25 @@
 #include <asm/hw_irq.h>
 #include <asm/kvm_book3s_asm.h>
 #include <asm/opal.h>
+#include <asm/cpuidle.h>
+#include <asm/mmu-hash64.h>
 
 #undef DEBUG
 
+/*
+ * Use unused space in the interrupt stack to save and restore
+ * registers for winkle support.
+ */
+#define _SDR1   GPR3
+#define _RPR    GPR4
+#define _SPURR  GPR5
+#define _PURR   GPR6
+#define _TSCR   GPR7
+#define _DSCR   GPR8
+#define _AMOR   GPR9
+#define _WORT   GPR10
+#define _WORC   GPR11
+
 /* Idle state entry routines */
 
 #define IDLE_STATE_ENTER_SEQ(IDLE_INST)                         \
@@ -37,8 +53,7 @@
 
 /*
  * Pass requested state in r3:
- *      0 - nap
- *      1 - sleep
+ *      r3 - PNV_THREAD_NAP/SLEEP/WINKLE
  *
  * To check IRQ_HAPPENED in r4
  *      0 - don't check
@@ -101,18 +116,105 @@ _GLOBAL(power7_powersave_common)
         std     r9,_MSR(r1)
         std     r1,PACAR1(r13)
 
-_GLOBAL(power7_enter_nap_mode)
+        /*
+         * Go to real mode to do the nap, as required by the architecture.
+         * Also, we need to be in real mode before setting hwthread_state,
+         * because as soon as we do that, another thread can switch
+         * the MMU context to the guest.
+         */
+        LOAD_REG_IMMEDIATE(r5, MSR_IDLE)
+        li      r6, MSR_RI
+        andc    r6, r9, r6
+        LOAD_REG_ADDR(r7, power7_enter_nap_mode)
+        mtmsrd  r6, 1           /* clear RI before setting SRR0/1 */
+        mtspr   SPRN_SRR0, r7
+        mtspr   SPRN_SRR1, r5
+        rfid
+
+        .globl  power7_enter_nap_mode
+power7_enter_nap_mode:
 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
         /* Tell KVM we're napping */
         li      r4,KVM_HWTHREAD_IN_NAP
         stb     r4,HSTATE_HWTHREAD_STATE(r13)
 #endif
-        cmpwi   cr0,r3,1
-        beq     2f
+        stb     r3,PACA_THREAD_IDLE_STATE(r13)
+        cmpwi   cr3,r3,PNV_THREAD_SLEEP
+        bge     cr3,2f
         IDLE_STATE_ENTER_SEQ(PPC_NAP)
         /* No return */
-2:      IDLE_STATE_ENTER_SEQ(PPC_SLEEP)
-        /* No return */
+2:
+        /* Sleep or winkle */
+        lbz     r7,PACA_THREAD_MASK(r13)
+        ld      r14,PACA_CORE_IDLE_STATE_PTR(r13)
+lwarx_loop1:
+        lwarx   r15,0,r14
+        andc    r15,r15,r7                      /* Clear thread bit */
+
+        andi.   r15,r15,PNV_CORE_IDLE_THREAD_BITS
+
+/*
+ * If cr0 = 0, then current thread is the last thread of the core entering
+ * sleep. Last thread needs to execute the hardware bug workaround code if
+ * required by the platform.
+ * Make the workaround call unconditionally here. The below branch call is
+ * patched out when the idle states are discovered if the platform does not
+ * require it.
+ */
+.global pnv_fastsleep_workaround_at_entry
+pnv_fastsleep_workaround_at_entry:
+        beq     fastsleep_workaround_at_entry
+
+        stwcx.  r15,0,r14
+        bne-    lwarx_loop1
+        isync
+
+common_enter: /* common code for all the threads entering sleep or winkle */
+        bgt     cr3,enter_winkle
+        IDLE_STATE_ENTER_SEQ(PPC_SLEEP)
+
+fastsleep_workaround_at_entry:
+        ori     r15,r15,PNV_CORE_IDLE_LOCK_BIT
+        stwcx.  r15,0,r14
+        bne-    lwarx_loop1
+        isync
+
+        /* Fast sleep workaround */
+        li      r3,1
+        li      r4,1
+        li      r0,OPAL_CONFIG_CPU_IDLE_STATE
+        bl      opal_call_realmode
+
+        /* Clear Lock bit */
+        li      r0,0
+        lwsync
+        stw     r0,0(r14)
+        b       common_enter
+
+enter_winkle:
+        /*
+         * Note all register i.e per-core, per-subcore or per-thread is saved
+         * here since any thread in the core might wake up first
+         */
+        mfspr   r3,SPRN_SDR1
+        std     r3,_SDR1(r1)
+        mfspr   r3,SPRN_RPR
+        std     r3,_RPR(r1)
+        mfspr   r3,SPRN_SPURR
+        std     r3,_SPURR(r1)
+        mfspr   r3,SPRN_PURR
+        std     r3,_PURR(r1)
+        mfspr   r3,SPRN_TSCR
+        std     r3,_TSCR(r1)
+        mfspr   r3,SPRN_DSCR
+        std     r3,_DSCR(r1)
+        mfspr   r3,SPRN_AMOR
+        std     r3,_AMOR(r1)
+        mfspr   r3,SPRN_WORT
+        std     r3,_WORT(r1)
+        mfspr   r3,SPRN_WORC
+        std     r3,_WORC(r1)
+        IDLE_STATE_ENTER_SEQ(PPC_WINKLE)
 
 _GLOBAL(power7_idle)
         /* Now check if user or arch enabled NAP mode */
@@ -125,48 +227,21 @@ _GLOBAL(power7_idle)
 
 _GLOBAL(power7_nap)
         mr      r4,r3
-        li      r3,0
+        li      r3,PNV_THREAD_NAP
         b       power7_powersave_common
         /* No return */
 
 _GLOBAL(power7_sleep)
-        li      r3,1
+        li      r3,PNV_THREAD_SLEEP
         li      r4,1
         b       power7_powersave_common
         /* No return */
 
-/*
- * Make opal call in realmode. This is a generic function to be called
- * from realmode from reset vector. It handles endianess.
- *
- * r13 - paca pointer
- * r1  - stack pointer
- * r3  - opal token
- */
-opal_call_realmode:
-        mflr    r12
-        std     r12,_LINK(r1)
-        ld      r2,PACATOC(r13)
-        /* Set opal return address */
-        LOAD_REG_ADDR(r0,return_from_opal_call)
-        mtlr    r0
-        /* Handle endian-ness */
-        li      r0,MSR_LE
-        mfmsr   r12
-        andc    r12,r12,r0
-        mtspr   SPRN_HSRR1,r12
-        mr      r0,r3                   /* Move opal token to r0 */
-        LOAD_REG_ADDR(r11,opal)
-        ld      r12,8(r11)
-        ld      r2,0(r11)
-        mtspr   SPRN_HSRR0,r12
-        hrfid
-
-return_from_opal_call:
-        FIXUP_ENDIAN
-        ld      r0,_LINK(r1)
-        mtlr    r0
-        blr
+_GLOBAL(power7_winkle)
+        li      r3,3
+        li      r4,1
+        b       power7_powersave_common
+        /* No return */
 
 #define CHECK_HMI_INTERRUPT                                             \
         mfspr   r0,SPRN_SRR1;                                           \
@@ -181,7 +256,7 @@ ALT_FTR_SECTION_END_NESTED_IFSET(CPU_FTR_ARCH_207S, 66);		\
         ld      r2,PACATOC(r13);                                        \
         ld      r1,PACAR1(r13);                                         \
         std     r3,ORIG_GPR3(r1);       /* Save original r3 */          \
-        li      r3,OPAL_HANDLE_HMI;     /* Pass opal token argument*/   \
+        li      r0,OPAL_HANDLE_HMI;     /* Pass opal token argument*/   \
         bl      opal_call_realmode;                                     \
         ld      r3,ORIG_GPR3(r1);       /* Restore original r3 */       \
 20:     nop;
@@ -190,16 +265,190 @@ ALT_FTR_SECTION_END_NESTED_IFSET(CPU_FTR_ARCH_207S, 66);		\
 _GLOBAL(power7_wakeup_tb_loss)
         ld      r2,PACATOC(r13);
         ld      r1,PACAR1(r13)
+        /*
+         * Before entering any idle state, the NVGPRs are saved in the stack
+         * and they are restored before switching to the process context. Hence
+         * until they are restored, they are free to be used.
+         *
+         * Save SRR1 in a NVGPR as it might be clobbered in opal_call_realmode
+         * (called in CHECK_HMI_INTERRUPT). SRR1 is required to determine the
+         * wakeup reason if we branch to kvm_start_guest.
+         */
 
+        mfspr   r16,SPRN_SRR1
 BEGIN_FTR_SECTION
         CHECK_HMI_INTERRUPT
 END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
+
+        lbz     r7,PACA_THREAD_MASK(r13)
+        ld      r14,PACA_CORE_IDLE_STATE_PTR(r13)
+lwarx_loop2:
+        lwarx   r15,0,r14
+        andi.   r9,r15,PNV_CORE_IDLE_LOCK_BIT
+        /*
+         * Lock bit is set in one of the 2 cases-
+         * a. In the sleep/winkle enter path, the last thread is executing
+         * fastsleep workaround code.
+         * b. In the wake up path, another thread is executing fastsleep
+         * workaround undo code or resyncing timebase or restoring context
+         * In either case loop until the lock bit is cleared.
+         */
+        bne     core_idle_lock_held
+
+        cmpwi   cr2,r15,0
+        lbz     r4,PACA_SUBCORE_SIBLING_MASK(r13)
+        and     r4,r4,r15
+        cmpwi   cr1,r4,0        /* Check if first in subcore */
+
+        /*
+         * At this stage
+         * cr1 - 0b0100 if first thread to wakeup in subcore
+         * cr2 - 0b0100 if first thread to wakeup in core
+         * cr3-  0b0010 if waking up from sleep or winkle
+         * cr4 - 0b0100 if waking up from winkle
+         */
+
+        or      r15,r15,r7              /* Set thread bit */
+
+        beq     cr1,first_thread_in_subcore
+
+        /* Not first thread in subcore to wake up */
+        stwcx.  r15,0,r14
+        bne-    lwarx_loop2
+        isync
+        b       common_exit
+
+core_idle_lock_held:
+        HMT_LOW
+core_idle_lock_loop:
+        lwz     r15,0(14)
+        andi.   r9,r15,PNV_CORE_IDLE_LOCK_BIT
+        bne     core_idle_lock_loop
+        HMT_MEDIUM
+        b       lwarx_loop2
+
+first_thread_in_subcore:
+        /* First thread in subcore to wakeup */
+        ori     r15,r15,PNV_CORE_IDLE_LOCK_BIT
+        stwcx.  r15,0,r14
+        bne-    lwarx_loop2
+        isync
+
+        /*
+         * If waking up from sleep, subcore state is not lost. Hence
+         * skip subcore state restore
+         */
+        bne     cr4,subcore_state_restored
+
+        /* Restore per-subcore state */
+        ld      r4,_SDR1(r1)
+        mtspr   SPRN_SDR1,r4
+        ld      r4,_RPR(r1)
+        mtspr   SPRN_RPR,r4
+        ld      r4,_AMOR(r1)
+        mtspr   SPRN_AMOR,r4
+
+subcore_state_restored:
+        /*
+         * Check if the thread is also the first thread in the core. If not,
+         * skip to clear_lock.
+         */
+        bne     cr2,clear_lock
+
+first_thread_in_core:
+
+        /*
+         * First thread in the core waking up from fastsleep. It needs to
+         * call the fastsleep workaround code if the platform requires it.
+         * Call it unconditionally here. The below branch instruction will
+         * be patched out when the idle states are discovered if platform
+         * does not require workaround.
+         */
+.global pnv_fastsleep_workaround_at_exit
+pnv_fastsleep_workaround_at_exit:
+        b       fastsleep_workaround_at_exit
+
+timebase_resync:
+        /* Do timebase resync if we are waking up from sleep. Use cr3 value
+         * set in exceptions-64s.S */
+        ble     cr3,clear_lock
         /* Time base re-sync */
-        li      r3,OPAL_RESYNC_TIMEBASE
+        li      r0,OPAL_RESYNC_TIMEBASE
         bl      opal_call_realmode;
-
         /* TODO: Check r3 for failure */
 
+        /*
+         * If waking up from sleep, per core state is not lost, skip to
+         * clear_lock.
+         */
+        bne     cr4,clear_lock
+
+        /* Restore per core state */
+        ld      r4,_TSCR(r1)
+        mtspr   SPRN_TSCR,r4
+        ld      r4,_WORC(r1)
+        mtspr   SPRN_WORC,r4
+
+clear_lock:
+        andi.   r15,r15,PNV_CORE_IDLE_THREAD_BITS
+        lwsync
+        stw     r15,0(r14)
+
+common_exit:
+        /*
+         * Common to all threads.
+         *
+         * If waking up from sleep, hypervisor state is not lost. Hence
+         * skip hypervisor state restore.
+         */
+        bne     cr4,hypervisor_state_restored
+
+        /* Waking up from winkle */
+
+        /* Restore per thread state */
+        bl      __restore_cpu_power8
+
+        /* Restore SLB  from PACA */
+        ld      r8,PACA_SLBSHADOWPTR(r13)
+
+        .rept   SLB_NUM_BOLTED
+        li      r3, SLBSHADOW_SAVEAREA
+        LDX_BE  r5, r8, r3
+        addi    r3, r3, 8
+        LDX_BE  r6, r8, r3
+        andis.  r7,r5,SLB_ESID_V@h
+        beq     1f
+        slbmte  r6,r5
+1:      addi    r8,r8,16
+        .endr
+
+        ld      r4,_SPURR(r1)
+        mtspr   SPRN_SPURR,r4
+        ld      r4,_PURR(r1)
+        mtspr   SPRN_PURR,r4
+        ld      r4,_DSCR(r1)
+        mtspr   SPRN_DSCR,r4
+        ld      r4,_WORT(r1)
+        mtspr   SPRN_WORT,r4
+
+hypervisor_state_restored:
+
+        li      r5,PNV_THREAD_RUNNING
+        stb     r5,PACA_THREAD_IDLE_STATE(r13)
+
+        mtspr   SPRN_SRR1,r16
+#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
+        li      r0,KVM_HWTHREAD_IN_KERNEL
+        stb     r0,HSTATE_HWTHREAD_STATE(r13)
+        /* Order setting hwthread_state vs. testing hwthread_req */
+        sync
+        lbz     r0,HSTATE_HWTHREAD_REQ(r13)
+        cmpwi   r0,0
+        beq     6f
+        b       kvm_start_guest
+6:
+#endif
+
         REST_NVGPRS(r1)
         REST_GPR(2, r1)
         ld      r3,_CCR(r1)
@@ -212,6 +461,13 @@ END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
         mtspr   SPRN_SRR0,r5
         rfid
 
+fastsleep_workaround_at_exit:
+        li      r3,1
+        li      r4,0
+        li      r0,OPAL_CONFIG_CPU_IDLE_STATE
+        bl      opal_call_realmode
+        b       timebase_resync
+
 /*
  * R3 here contains the value that will be returned to the caller
  * of power7_nap.
diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c
index 8b2d2dc8ef10..8ec017cb4446 100644
--- a/arch/powerpc/kernel/smp.c
+++ b/arch/powerpc/kernel/smp.c
@@ -700,7 +700,6 @@ void start_secondary(void *unused)
         smp_store_cpu_info(cpu);
         set_dec(tb_ticks_per_jiffy);
         preempt_disable();
-        cpu_callin_map[cpu] = 1;
 
         if (smp_ops->setup_cpu)
                 smp_ops->setup_cpu(cpu);
@@ -739,6 +738,14 @@ void start_secondary(void *unused)
         notify_cpu_starting(cpu);
         set_cpu_online(cpu, true);
 
+        /*
+         * CPU must be marked active and online before we signal back to the
+         * master, because the scheduler needs to see the cpu_online and
+         * cpu_active bits set.
+         */
+        smp_wmb();
+        cpu_callin_map[cpu] = 1;
+
         local_irq_enable();
 
         cpu_startup_entry(CPUHP_ONLINE);