3 files changed, 300 insertions, 61 deletions

diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h
index 948f01a04cc3..8e956a0b6e85 100644
--- a/arch/powerpc/include/asm/paca.h
+++ b/arch/powerpc/include/asm/paca.h
@@ -116,8 +116,11 @@ struct paca_struct {
         /* Shared by all threads of a core -- points to tcd of first thread */
         struct tlb_core_data *tcd_ptr;
 
-        /* We can have up to 3 levels of reentrancy in the TLB miss handler */
-        u64 extlb[3][EX_TLB_SIZE / sizeof(u64)];
+        /*
+         * We can have up to 3 levels of reentrancy in the TLB miss handler,
+         * in each of four exception levels (normal, crit, mcheck, debug).
+         */
+        u64 extlb[12][EX_TLB_SIZE / sizeof(u64)];
         u64 exmc[8];            /* used for machine checks */
         u64 excrit[8];          /* used for crit interrupts */
         u64 exdbg[8];           /* used for debug interrupts */
diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S
index 2beb5bd0728b..c1bee3ce9d1f 100644
--- a/arch/powerpc/kernel/exceptions-64e.S
+++ b/arch/powerpc/kernel/exceptions-64e.S
@@ -34,20 +34,250 @@
  *     special interrupts from within a non-standard level will probably
  *     blow you up
  */
-#define SPECIAL_EXC_FRAME_SIZE  INT_FRAME_SIZE
+#define SPECIAL_EXC_SRR0        0
+#define SPECIAL_EXC_SRR1        1
+#define SPECIAL_EXC_SPRG_GEN    2
+#define SPECIAL_EXC_SPRG_TLB    3
+#define SPECIAL_EXC_MAS0        4
+#define SPECIAL_EXC_MAS1        5
+#define SPECIAL_EXC_MAS2        6
+#define SPECIAL_EXC_MAS3        7
+#define SPECIAL_EXC_MAS6        8
+#define SPECIAL_EXC_MAS7        9
+#define SPECIAL_EXC_MAS5        10      /* E.HV only */
+#define SPECIAL_EXC_MAS8        11      /* E.HV only */
+#define SPECIAL_EXC_IRQHAPPENED 12
+#define SPECIAL_EXC_DEAR        13
+#define SPECIAL_EXC_ESR         14
+#define SPECIAL_EXC_SOFTE       15
+#define SPECIAL_EXC_CSRR0       16
+#define SPECIAL_EXC_CSRR1       17
+/* must be even to keep 16-byte stack alignment */
+#define SPECIAL_EXC_END         18
+
+#define SPECIAL_EXC_FRAME_SIZE  (INT_FRAME_SIZE + SPECIAL_EXC_END * 8)
+#define SPECIAL_EXC_FRAME_OFFS  (INT_FRAME_SIZE - 288)
+
+#define SPECIAL_EXC_STORE(reg, name) \
+        std     reg, (SPECIAL_EXC_##name * 8 + SPECIAL_EXC_FRAME_OFFS)(r1)
+
+#define SPECIAL_EXC_LOAD(reg, name) \
+        ld      reg, (SPECIAL_EXC_##name * 8 + SPECIAL_EXC_FRAME_OFFS)(r1)
+
+special_reg_save:
+        lbz     r9,PACAIRQHAPPENED(r13)
+        RECONCILE_IRQ_STATE(r3,r4)
 
-/* Now we only store something to exception thread info */
-#define EXC_LEVEL_EXCEPTION_PROLOG(type)                                \
-        ld      r14,PACAKSAVE(r13);                                     \
-        CURRENT_THREAD_INFO(r14, r14);                                  \
-        CURRENT_THREAD_INFO(r15, r1);                                   \
-        ld      r10,TI_FLAGS(r14);                                      \
-        std     r10,TI_FLAGS(r15);                                      \
-        ld      r10,TI_PREEMPT(r14);                                    \
-        std     r10,TI_PREEMPT(r15);                                    \
-        ld      r10,TI_TASK(r14);                                       \
-        std     r10,TI_TASK(r15);
+        /*
+         * We only need (or have stack space) to save this stuff if
+         * we interrupted the kernel.
+         */
+        ld      r3,_MSR(r1)
+        andi.   r3,r3,MSR_PR
+        bnelr
+
+        /* Copy info into temporary exception thread info */
+        ld      r11,PACAKSAVE(r13)
+        CURRENT_THREAD_INFO(r11, r11)
+        CURRENT_THREAD_INFO(r12, r1)
+        ld      r10,TI_FLAGS(r11)
+        std     r10,TI_FLAGS(r12)
+        ld      r10,TI_PREEMPT(r11)
+        std     r10,TI_PREEMPT(r12)
+        ld      r10,TI_TASK(r11)
+        std     r10,TI_TASK(r12)
+
+        /*
+         * Advance to the next TLB exception frame for handler
+         * types that don't do it automatically.
+         */
+        LOAD_REG_ADDR(r11,extlb_level_exc)
+        lwz     r12,0(r11)
+        mfspr   r10,SPRN_SPRG_TLB_EXFRAME
+        add     r10,r10,r12
+        mtspr   SPRN_SPRG_TLB_EXFRAME,r10
+
+        /*
+         * Save registers needed to allow nesting of certain exceptions
+         * (such as TLB misses) inside special exception levels
+         */
+        mfspr   r10,SPRN_SRR0
+        SPECIAL_EXC_STORE(r10,SRR0)
+        mfspr   r10,SPRN_SRR1
+        SPECIAL_EXC_STORE(r10,SRR1)
+        mfspr   r10,SPRN_SPRG_GEN_SCRATCH
+        SPECIAL_EXC_STORE(r10,SPRG_GEN)
+        mfspr   r10,SPRN_SPRG_TLB_SCRATCH
+        SPECIAL_EXC_STORE(r10,SPRG_TLB)
+        mfspr   r10,SPRN_MAS0
+        SPECIAL_EXC_STORE(r10,MAS0)
+        mfspr   r10,SPRN_MAS1
+        SPECIAL_EXC_STORE(r10,MAS1)
+        mfspr   r10,SPRN_MAS2
+        SPECIAL_EXC_STORE(r10,MAS2)
+        mfspr   r10,SPRN_MAS3
+        SPECIAL_EXC_STORE(r10,MAS3)
+        mfspr   r10,SPRN_MAS6
+        SPECIAL_EXC_STORE(r10,MAS6)
+        mfspr   r10,SPRN_MAS7
+        SPECIAL_EXC_STORE(r10,MAS7)
+BEGIN_FTR_SECTION
+        mfspr   r10,SPRN_MAS5
+        SPECIAL_EXC_STORE(r10,MAS5)
+        mfspr   r10,SPRN_MAS8
+        SPECIAL_EXC_STORE(r10,MAS8)
+
+        /* MAS5/8 could have inappropriate values if we interrupted KVM code */
+        li      r10,0
+        mtspr   SPRN_MAS5,r10
+        mtspr   SPRN_MAS8,r10
+END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
+        SPECIAL_EXC_STORE(r9,IRQHAPPENED)
+
+        mfspr   r10,SPRN_DEAR
+        SPECIAL_EXC_STORE(r10,DEAR)
+        mfspr   r10,SPRN_ESR
+        SPECIAL_EXC_STORE(r10,ESR)
+
+        lbz     r10,PACASOFTIRQEN(r13)
+        SPECIAL_EXC_STORE(r10,SOFTE)
+        ld      r10,_NIP(r1)
+        SPECIAL_EXC_STORE(r10,CSRR0)
+        ld      r10,_MSR(r1)
+        SPECIAL_EXC_STORE(r10,CSRR1)
+
+        blr
+
+ret_from_level_except:
+        ld      r3,_MSR(r1)
+        andi.   r3,r3,MSR_PR
+        beq     1f
+        b       ret_from_except
+1:
+
+        LOAD_REG_ADDR(r11,extlb_level_exc)
+        lwz     r12,0(r11)
+        mfspr   r10,SPRN_SPRG_TLB_EXFRAME
+        sub     r10,r10,r12
+        mtspr   SPRN_SPRG_TLB_EXFRAME,r10
+
+        /*
+         * It's possible that the special level exception interrupted a
+         * TLB miss handler, and inserted the same entry that the
+         * interrupted handler was about to insert.  On CPUs without TLB
+         * write conditional, this can result in a duplicate TLB entry.
+         * Wipe all non-bolted entries to be safe.
+         *
+         * Note that this doesn't protect against any TLB misses
+         * we may take accessing the stack from here to the end of
+         * the special level exception.  It's not clear how we can
+         * reasonably protect against that, but only CPUs with
+         * neither TLB write conditional nor bolted kernel memory
+         * are affected.  Do any such CPUs even exist?
+         */
+        PPC_TLBILX_ALL(0,R0)
+
+        REST_NVGPRS(r1)
+
+        SPECIAL_EXC_LOAD(r10,SRR0)
+        mtspr   SPRN_SRR0,r10
+        SPECIAL_EXC_LOAD(r10,SRR1)
+        mtspr   SPRN_SRR1,r10
+        SPECIAL_EXC_LOAD(r10,SPRG_GEN)
+        mtspr   SPRN_SPRG_GEN_SCRATCH,r10
+        SPECIAL_EXC_LOAD(r10,SPRG_TLB)
+        mtspr   SPRN_SPRG_TLB_SCRATCH,r10
+        SPECIAL_EXC_LOAD(r10,MAS0)
+        mtspr   SPRN_MAS0,r10
+        SPECIAL_EXC_LOAD(r10,MAS1)
+        mtspr   SPRN_MAS1,r10
+        SPECIAL_EXC_LOAD(r10,MAS2)
+        mtspr   SPRN_MAS2,r10
+        SPECIAL_EXC_LOAD(r10,MAS3)
+        mtspr   SPRN_MAS3,r10
+        SPECIAL_EXC_LOAD(r10,MAS6)
+        mtspr   SPRN_MAS6,r10
+        SPECIAL_EXC_LOAD(r10,MAS7)
+        mtspr   SPRN_MAS7,r10
+BEGIN_FTR_SECTION
+        SPECIAL_EXC_LOAD(r10,MAS5)
+        mtspr   SPRN_MAS5,r10
+        SPECIAL_EXC_LOAD(r10,MAS8)
+        mtspr   SPRN_MAS8,r10
+END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
+
+        lbz     r6,PACASOFTIRQEN(r13)
+        ld      r5,SOFTE(r1)
+
+        /* Interrupts had better not already be enabled... */
+        twnei   r6,0
+
+        cmpwi   cr0,r5,0
+        beq     1f
 
+        TRACE_ENABLE_INTS
+        stb     r5,PACASOFTIRQEN(r13)
+1:
+        /*
+         * Restore PACAIRQHAPPENED rather than setting it based on
+         * the return MSR[EE], since we could have interrupted
+         * __check_irq_replay() or other inconsistent transitory
+         * states that must remain that way.
+         */
+        SPECIAL_EXC_LOAD(r10,IRQHAPPENED)
+        stb     r10,PACAIRQHAPPENED(r13)
+
+        SPECIAL_EXC_LOAD(r10,DEAR)
+        mtspr   SPRN_DEAR,r10
+        SPECIAL_EXC_LOAD(r10,ESR)
+        mtspr   SPRN_ESR,r10
+
+        stdcx.  r0,0,r1         /* to clear the reservation */
+
+        REST_4GPRS(2, r1)
+        REST_4GPRS(6, r1)
+
+        ld      r10,_CTR(r1)
+        ld      r11,_XER(r1)
+        mtctr   r10
+        mtxer   r11
+
+        blr
+
+.macro ret_from_level srr0 srr1 paca_ex scratch
+        bl      ret_from_level_except
+
+        ld      r10,_LINK(r1)
+        ld      r11,_CCR(r1)
+        ld      r0,GPR13(r1)
+        mtlr    r10
+        mtcr    r11
+
+        ld      r10,GPR10(r1)
+        ld      r11,GPR11(r1)
+        ld      r12,GPR12(r1)
+        mtspr   \scratch,r0
+
+        std     r10,\paca_ex+EX_R10(r13);
+        std     r11,\paca_ex+EX_R11(r13);
+        ld      r10,_NIP(r1)
+        ld      r11,_MSR(r1)
+        ld      r0,GPR0(r1)
+        ld      r1,GPR1(r1)
+        mtspr   \srr0,r10
+        mtspr   \srr1,r11
+        ld      r10,\paca_ex+EX_R10(r13)
+        ld      r11,\paca_ex+EX_R11(r13)
+        mfspr   r13,\scratch
+.endm
+
+ret_from_crit_except:
+        ret_from_level SPRN_CSRR0 SPRN_CSRR1 PACA_EXCRIT SPRN_SPRG_CRIT_SCRATCH
+        rfci
+
+ret_from_mc_except:
+        ret_from_level SPRN_MCSRR0 SPRN_MCSRR1 PACA_EXMC SPRN_SPRG_MC_SCRATCH
+        rfmci
 
 /* Exception prolog code for all exceptions */
 #define EXCEPTION_PROLOG(n, intnum, type, addition)                         \
@@ -81,22 +311,19 @@
 
 #define CRIT_SET_KSTACK                                                     \
         ld      r1,PACA_CRIT_STACK(r13);                                    \
-        subi    r1,r1,SPECIAL_EXC_FRAME_SIZE;                               \
-        EXC_LEVEL_EXCEPTION_PROLOG(CRIT);
+        subi    r1,r1,SPECIAL_EXC_FRAME_SIZE
 #define SPRN_CRIT_SRR0  SPRN_CSRR0
 #define SPRN_CRIT_SRR1  SPRN_CSRR1
 
 #define DBG_SET_KSTACK                                                      \
         ld      r1,PACA_DBG_STACK(r13);                                     \
-        subi    r1,r1,SPECIAL_EXC_FRAME_SIZE;                               \
-        EXC_LEVEL_EXCEPTION_PROLOG(DBG);
+        subi    r1,r1,SPECIAL_EXC_FRAME_SIZE
 #define SPRN_DBG_SRR0   SPRN_DSRR0
 #define SPRN_DBG_SRR1   SPRN_DSRR1
 
 #define MC_SET_KSTACK                                                       \
         ld      r1,PACA_MC_STACK(r13);                                      \
-        subi    r1,r1,SPECIAL_EXC_FRAME_SIZE;                               \
-        EXC_LEVEL_EXCEPTION_PROLOG(MC);
+        subi    r1,r1,SPECIAL_EXC_FRAME_SIZE
 #define SPRN_MC_SRR0    SPRN_MCSRR0
 #define SPRN_MC_SRR1    SPRN_MCSRR1
 
@@ -322,27 +549,25 @@ interrupt_end_book3e:
         START_EXCEPTION(critical_input);
         CRIT_EXCEPTION_PROLOG(0x100, BOOKE_INTERRUPT_CRITICAL,
                               PROLOG_ADDITION_NONE)
-//      EXCEPTION_COMMON_CRIT(0x100)
-//      INTS_DISABLE
-//      bl      special_reg_save_crit
-//      CHECK_NAPPING();
-//      addi    r3,r1,STACK_FRAME_OVERHEAD
-//      bl      .critical_exception
-//      b       ret_from_crit_except
-        b       .
+        EXCEPTION_COMMON_CRIT(0x100)
+        bl      .save_nvgprs
+        bl      special_reg_save
+        CHECK_NAPPING();
+        addi    r3,r1,STACK_FRAME_OVERHEAD
+        bl      .unknown_exception
+        b       ret_from_crit_except
 
 /* Machine Check Interrupt */
         START_EXCEPTION(machine_check);
         MC_EXCEPTION_PROLOG(0x000, BOOKE_INTERRUPT_MACHINE_CHECK,
                             PROLOG_ADDITION_NONE)
-//      EXCEPTION_COMMON_MC(0x000)
-//      INTS_DISABLE
-//      bl      special_reg_save_mc
-//      addi    r3,r1,STACK_FRAME_OVERHEAD
-//      CHECK_NAPPING();
-//      bl      .machine_check_exception
-//      b       ret_from_mc_except
-        b       .
+        EXCEPTION_COMMON_MC(0x000)
+        bl      .save_nvgprs
+        bl      special_reg_save
+        CHECK_NAPPING();
+        addi    r3,r1,STACK_FRAME_OVERHEAD
+        bl      .machine_check_exception
+        b       ret_from_mc_except
 
 /* Data Storage Interrupt */
         START_EXCEPTION(data_storage)
@@ -461,14 +686,17 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
         START_EXCEPTION(watchdog);
         CRIT_EXCEPTION_PROLOG(0x9f0, BOOKE_INTERRUPT_WATCHDOG,
                               PROLOG_ADDITION_NONE)
-//      EXCEPTION_COMMON_CRIT(0x9f0)
-//      INTS_DISABLE
-//      bl      special_reg_save_crit
-//      CHECK_NAPPING();
-//      addi    r3,r1,STACK_FRAME_OVERHEAD
-//      bl      .unknown_exception
-//      b       ret_from_crit_except
-        b       .
+        EXCEPTION_COMMON_CRIT(0x9f0)
+        bl      .save_nvgprs
+        bl      special_reg_save
+        CHECK_NAPPING();
+        addi    r3,r1,STACK_FRAME_OVERHEAD
+#ifdef CONFIG_BOOKE_WDT
+        bl      .WatchdogException
+#else
+        bl      .unknown_exception
+#endif
+        b       ret_from_crit_except
 
 /* System Call Interrupt */
         START_EXCEPTION(system_call)
@@ -541,7 +769,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
          */
         mfspr   r14,SPRN_DBSR
         EXCEPTION_COMMON_CRIT(0xd00)
-        INTS_DISABLE
         std     r14,_DSISR(r1)
         addi    r3,r1,STACK_FRAME_OVERHEAD
         mr      r4,r14
@@ -634,14 +861,13 @@ kernel_dbg_exc:
         START_EXCEPTION(doorbell_crit);
         CRIT_EXCEPTION_PROLOG(0x2a0, BOOKE_INTERRUPT_DOORBELL_CRITICAL,
                               PROLOG_ADDITION_NONE)
-//      EXCEPTION_COMMON_CRIT(0x2a0)
-//      INTS_DISABLE
-//      bl      special_reg_save_crit
-//      CHECK_NAPPING();
-//      addi    r3,r1,STACK_FRAME_OVERHEAD
-//      bl      .doorbell_critical_exception
-//      b       ret_from_crit_except
-        b       .
+        EXCEPTION_COMMON_CRIT(0x2a0)
+        bl      .save_nvgprs
+        bl      special_reg_save
+        CHECK_NAPPING();
+        addi    r3,r1,STACK_FRAME_OVERHEAD
+        bl      .unknown_exception
+        b       ret_from_crit_except
 
 /*
  *      Guest doorbell interrupt
@@ -661,14 +887,13 @@ kernel_dbg_exc:
         START_EXCEPTION(guest_doorbell_crit);
         CRIT_EXCEPTION_PROLOG(0x2e0, BOOKE_INTERRUPT_GUEST_DBELL_CRIT,
                               PROLOG_ADDITION_NONE)
-//      EXCEPTION_COMMON_CRIT(0x2e0)
-//      INTS_DISABLE
-//      bl      special_reg_save_crit
-//      CHECK_NAPPING();
-//      addi    r3,r1,STACK_FRAME_OVERHEAD
-//      bl      .guest_doorbell_critical_exception
-//      b       ret_from_crit_except
-        b       .
+        EXCEPTION_COMMON_CRIT(0x2e0)
+        bl      .save_nvgprs
+        bl      special_reg_save
+        CHECK_NAPPING();
+        addi    r3,r1,STACK_FRAME_OVERHEAD
+        bl      .unknown_exception
+        b       ret_from_crit_except
 
 /* Hypervisor call */
         START_EXCEPTION(hypercall);
diff --git a/arch/powerpc/mm/tlb_nohash.c b/arch/powerpc/mm/tlb_nohash.c
index b37a58e1c92d..ae3d5b799b90 100644
--- a/arch/powerpc/mm/tlb_nohash.c
+++ b/arch/powerpc/mm/tlb_nohash.c
@@ -144,6 +144,15 @@ int mmu_vmemmap_psize;		/* Page size used for the virtual mem map */
 int book3e_htw_mode;            /* HW tablewalk?  Value is PPC_HTW_* */
 unsigned long linear_map_top;   /* Top of linear mapping */
 
+
+/*
+ * Number of bytes to add to SPRN_SPRG_TLB_EXFRAME on crit/mcheck/debug
+ * exceptions.  This is used for bolted and e6500 TLB miss handlers which
+ * do not modify this SPRG in the TLB miss code; for other TLB miss handlers,
+ * this is set to zero.
+ */
+int extlb_level_exc;
+
 #endif /* CONFIG_PPC64 */
 
 #ifdef CONFIG_PPC_FSL_BOOK3E
@@ -559,6 +568,7 @@ static void setup_mmu_htw(void)
                 break;
 #ifdef CONFIG_PPC_FSL_BOOK3E
         case PPC_HTW_E6500:
+                extlb_level_exc = EX_TLB_SIZE;
                 patch_exception(0x1c0, exc_data_tlb_miss_e6500_book3e);
                 patch_exception(0x1e0, exc_instruction_tlb_miss_e6500_book3e);
                 break;
@@ -652,6 +662,7 @@ static void __early_init_mmu(int boot_cpu)
                 memblock_enforce_memory_limit(linear_map_top);
 
                 if (book3e_htw_mode == PPC_HTW_NONE) {
+                        extlb_level_exc = EX_TLB_SIZE;
                         patch_exception(0x1c0, exc_data_tlb_miss_bolted_book3e);
                         patch_exception(0x1e0,
                                 exc_instruction_tlb_miss_bolted_book3e);