powerpc/booke64: Critical and machine check exception support

Add special state saving for critical and machine check exceptions.

Most of this code could be used to handle debug exceptions taken from
kernel space, but actually doing so is outside the scope of this patch.

The various critical and machine check exceptions now point to their
real handlers, rather than hanging the kernel.

Signed-off-by: Scott Wood <scottwood@freescale.com>

3 files changed, 300 insertions, 61 deletions

diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h
index 948f01a04cc3..8e956a0b6e85 100644
--- a/arch/powerpc/include/asm/paca.h
+++ b/arch/powerpc/include/asm/paca.h
@@ -116,8 +116,11 @@ struct paca_struct {
         /* Shared by all threads of a core -- points to tcd of first thread */
         struct tlb_core_data *tcd_ptr;
 
-        /* We can have up to 3 levels of reentrancy in the TLB miss handler */
-        u64 extlb[3][EX_TLB_SIZE / sizeof(u64)];
+        /*
+         * We can have up to 3 levels of reentrancy in the TLB miss handler,
+         * in each of four exception levels (normal, crit, mcheck, debug).
+         */
+        u64 extlb[12][EX_TLB_SIZE / sizeof(u64)];
         u64 exmc[8];            /* used for machine checks */
         u64 excrit[8];          /* used for crit interrupts */
         u64 exdbg[8];           /* used for debug interrupts */
diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S
index 2beb5bd0728b..c1bee3ce9d1f 100644
--- a/arch/powerpc/kernel/exceptions-64e.S
+++ b/arch/powerpc/kernel/exceptions-64e.S
@@ -34,20 +34,250 @@
  *     special interrupts from within a non-standard level will probably
  *     blow you up
  */
-#define SPECIAL_EXC_FRAME_SIZE  INT_FRAME_SIZE
+#define SPECIAL_EXC_SRR0        0
+#define SPECIAL_EXC_SRR1        1
+#define SPECIAL_EXC_SPRG_GEN    2
+#define SPECIAL_EXC_SPRG_TLB    3
+#define SPECIAL_EXC_MAS0        4
+#define SPECIAL_EXC_MAS1        5
+#define SPECIAL_EXC_MAS2        6
+#define SPECIAL_EXC_MAS3        7
+#define SPECIAL_EXC_MAS6        8
+#define SPECIAL_EXC_MAS7        9
+#define SPECIAL_EXC_MAS5        10      /* E.HV only */
+#define SPECIAL_EXC_MAS8        11      /* E.HV only */
+#define SPECIAL_EXC_IRQHAPPENED 12
+#define SPECIAL_EXC_DEAR        13
+#define SPECIAL_EXC_ESR         14
+#define SPECIAL_EXC_SOFTE       15
+#define SPECIAL_EXC_CSRR0       16
+#define SPECIAL_EXC_CSRR1       17
+/* must be even to keep 16-byte stack alignment */
+#define SPECIAL_EXC_END         18
+
+#define SPECIAL_EXC_FRAME_SIZE  (INT_FRAME_SIZE + SPECIAL_EXC_END * 8)
+#define SPECIAL_EXC_FRAME_OFFS  (INT_FRAME_SIZE - 288)
+
+#define SPECIAL_EXC_STORE(reg, name) \
+        std     reg, (SPECIAL_EXC_##name * 8 + SPECIAL_EXC_FRAME_OFFS)(r1)
+
+#define SPECIAL_EXC_LOAD(reg, name) \
+        ld      reg, (SPECIAL_EXC_##name * 8 + SPECIAL_EXC_FRAME_OFFS)(r1)
+
+special_reg_save:
+        lbz     r9,PACAIRQHAPPENED(r13)
+        RECONCILE_IRQ_STATE(r3,r4)
 
-/* Now we only store something to exception thread info */
-#define EXC_LEVEL_EXCEPTION_PROLOG(type)                                \
-        ld      r14,PACAKSAVE(r13);                                     \
-        CURRENT_THREAD_INFO(r14, r14);                                  \
-        CURRENT_THREAD_INFO(r15, r1);                                   \
-        ld      r10,TI_FLAGS(r14);                                      \
-        std     r10,TI_FLAGS(r15);                                      \
-        ld      r10,TI_PREEMPT(r14);                                    \
-        std     r10,TI_PREEMPT(r15);                                    \
-        ld      r10,TI_TASK(r14);                                       \
-        std     r10,TI_TASK(r15);
+        /*
+         * We only need (or have stack space) to save this stuff if
+         * we interrupted the kernel.
+         */
+        ld      r3,_MSR(r1)
+        andi.   r3,r3,MSR_PR
+        bnelr
+
+        /* Copy info into temporary exception thread info */
+        ld      r11,PACAKSAVE(r13)
+        CURRENT_THREAD_INFO(r11, r11)
+        CURRENT_THREAD_INFO(r12, r1)
+        ld      r10,TI_FLAGS(r11)
+        std     r10,TI_FLAGS(r12)
+        ld      r10,TI_PREEMPT(r11)
+        std     r10,TI_PREEMPT(r12)
+        ld      r10,TI_TASK(r11)
+        std     r10,TI_TASK(r12)
+
+        /*
+         * Advance to the next TLB exception frame for handler
+         * types that don't do it automatically.
+         */
+        LOAD_REG_ADDR(r11,extlb_level_exc)
+        lwz     r12,0(r11)
+        mfspr   r10,SPRN_SPRG_TLB_EXFRAME
+        add     r10,r10,r12
+        mtspr   SPRN_SPRG_TLB_EXFRAME,r10
+
+        /*
+         * Save registers needed to allow nesting of certain exceptions
+         * (such as TLB misses) inside special exception levels
+         */
+        mfspr   r10,SPRN_SRR0
+        SPECIAL_EXC_STORE(r10,SRR0)
+        mfspr   r10,SPRN_SRR1
+        SPECIAL_EXC_STORE(r10,SRR1)
+        mfspr   r10,SPRN_SPRG_GEN_SCRATCH
+        SPECIAL_EXC_STORE(r10,SPRG_GEN)
+        mfspr   r10,SPRN_SPRG_TLB_SCRATCH
+        SPECIAL_EXC_STORE(r10,SPRG_TLB)
+        mfspr   r10,SPRN_MAS0
+        SPECIAL_EXC_STORE(r10,MAS0)
+        mfspr   r10,SPRN_MAS1
+        SPECIAL_EXC_STORE(r10,MAS1)
+        mfspr   r10,SPRN_MAS2
+        SPECIAL_EXC_STORE(r10,MAS2)
+        mfspr   r10,SPRN_MAS3
+        SPECIAL_EXC_STORE(r10,MAS3)
+        mfspr   r10,SPRN_MAS6
+        SPECIAL_EXC_STORE(r10,MAS6)
+        mfspr   r10,SPRN_MAS7
+        SPECIAL_EXC_STORE(r10,MAS7)
+BEGIN_FTR_SECTION
+        mfspr   r10,SPRN_MAS5
+        SPECIAL_EXC_STORE(r10,MAS5)
+        mfspr   r10,SPRN_MAS8
+        SPECIAL_EXC_STORE(r10,MAS8)
+
+        /* MAS5/8 could have inappropriate values if we interrupted KVM code */
+        li      r10,0
+        mtspr   SPRN_MAS5,r10
+        mtspr   SPRN_MAS8,r10
+END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
+        SPECIAL_EXC_STORE(r9,IRQHAPPENED)
+
+        mfspr   r10,SPRN_DEAR
+        SPECIAL_EXC_STORE(r10,DEAR)
+        mfspr   r10,SPRN_ESR
+        SPECIAL_EXC_STORE(r10,ESR)
+
+        lbz     r10,PACASOFTIRQEN(r13)
+        SPECIAL_EXC_STORE(r10,SOFTE)
+        ld      r10,_NIP(r1)
+        SPECIAL_EXC_STORE(r10,CSRR0)
+        ld      r10,_MSR(r1)
+        SPECIAL_EXC_STORE(r10,CSRR1)
+
+        blr
+
+ret_from_level_except:
+        ld      r3,_MSR(r1)
+        andi.   r3,r3,MSR_PR
+        beq     1f
+        b       ret_from_except
+1:
+
+        LOAD_REG_ADDR(r11,extlb_level_exc)
+        lwz     r12,0(r11)
+        mfspr   r10,SPRN_SPRG_TLB_EXFRAME
+        sub     r10,r10,r12
+        mtspr   SPRN_SPRG_TLB_EXFRAME,r10
+
+        /*
+         * It's possible that the special level exception interrupted a
+         * TLB miss handler, and inserted the same entry that the
+         * interrupted handler was about to insert.  On CPUs without TLB
+         * write conditional, this can result in a duplicate TLB entry.
+         * Wipe all non-bolted entries to be safe.
+         *
+         * Note that this doesn't protect against any TLB misses
+         * we may take accessing the stack from here to the end of
+         * the special level exception.  It's not clear how we can
+         * reasonably protect against that, but only CPUs with
+         * neither TLB write conditional nor bolted kernel memory
+         * are affected.  Do any such CPUs even exist?
+         */
+        PPC_TLBILX_ALL(0,R0)
+
+        REST_NVGPRS(r1)
+
+        SPECIAL_EXC_LOAD(r10,SRR0)
+        mtspr   SPRN_SRR0,r10
+        SPECIAL_EXC_LOAD(r10,SRR1)
+        mtspr   SPRN_SRR1,r10
+        SPECIAL_EXC_LOAD(r10,SPRG_GEN)
+        mtspr   SPRN_SPRG_GEN_SCRATCH,r10
+        SPECIAL_EXC_LOAD(r10,SPRG_TLB)
+        mtspr   SPRN_SPRG_TLB_SCRATCH,r10
+        SPECIAL_EXC_LOAD(r10,MAS0)
+        mtspr   SPRN_MAS0,r10
+        SPECIAL_EXC_LOAD(r10,MAS1)
+        mtspr   SPRN_MAS1,r10
+        SPECIAL_EXC_LOAD(r10,MAS2)
+        mtspr   SPRN_MAS2,r10
+        SPECIAL_EXC_LOAD(r10,MAS3)
+        mtspr   SPRN_MAS3,r10
+        SPECIAL_EXC_LOAD(r10,MAS6)
+        mtspr   SPRN_MAS6,r10
+        SPECIAL_EXC_LOAD(r10,MAS7)
+        mtspr   SPRN_MAS7,r10
+BEGIN_FTR_SECTION
+        SPECIAL_EXC_LOAD(r10,MAS5)
+        mtspr   SPRN_MAS5,r10
+        SPECIAL_EXC_LOAD(r10,MAS8)
+        mtspr   SPRN_MAS8,r10
+END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
+
+        lbz     r6,PACASOFTIRQEN(r13)
+        ld      r5,SOFTE(r1)
+
+        /* Interrupts had better not already be enabled... */
+        twnei   r6,0
+
+        cmpwi   cr0,r5,0
+        beq     1f
 
+        TRACE_ENABLE_INTS
+        stb     r5,PACASOFTIRQEN(r13)
+1:
+        /*
+         * Restore PACAIRQHAPPENED rather than setting it based on
+         * the return MSR[EE], since we could have interrupted
+         * __check_irq_replay() or other inconsistent transitory
+         * states that must remain that way.
+         */
+        SPECIAL_EXC_LOAD(r10,IRQHAPPENED)
+        stb     r10,PACAIRQHAPPENED(r13)
+
+        SPECIAL_EXC_LOAD(r10,DEAR)
+        mtspr   SPRN_DEAR,r10
+        SPECIAL_EXC_LOAD(r10,ESR)
+        mtspr   SPRN_ESR,r10
+
+        stdcx.  r0,0,r1         /* to clear the reservation */
+
+        REST_4GPRS(2, r1)
+        REST_4GPRS(6, r1)
+
+        ld      r10,_CTR(r1)
+        ld      r11,_XER(r1)
+        mtctr   r10
+        mtxer   r11
+
+        blr
+
+.macro ret_from_level srr0 srr1 paca_ex scratch
+        bl      ret_from_level_except
+
+        ld      r10,_LINK(r1)
+        ld      r11,_CCR(r1)
+        ld      r0,GPR13(r1)
+        mtlr    r10
+        mtcr    r11
+
+        ld      r10,GPR10(r1)
+        ld      r11,GPR11(r1)
+        ld      r12,GPR12(r1)
+        mtspr   \scratch,r0
+
+        std     r10,\paca_ex+EX_R10(r13);
+        std     r11,\paca_ex+EX_R11(r13);
+        ld      r10,_NIP(r1)
+        ld      r11,_MSR(r1)
+        ld      r0,GPR0(r1)
+        ld      r1,GPR1(r1)
+        mtspr   \srr0,r10
+        mtspr   \srr1,r11
+        ld      r10,\paca_ex+EX_R10(r13)
+        ld      r11,\paca_ex+EX_R11(r13)
+        mfspr   r13,\scratch
+.endm
+
+ret_from_crit_except:
+        ret_from_level SPRN_CSRR0 SPRN_CSRR1 PACA_EXCRIT SPRN_SPRG_CRIT_SCRATCH
+        rfci
+
+ret_from_mc_except:
+        ret_from_level SPRN_MCSRR0 SPRN_MCSRR1 PACA_EXMC SPRN_SPRG_MC_SCRATCH
+        rfmci
 
 /* Exception prolog code for all exceptions */
 #define EXCEPTION_PROLOG(n, intnum, type, addition)                         \
@@ -81,22 +311,19 @@
 
 #define CRIT_SET_KSTACK                                                     \
         ld      r1,PACA_CRIT_STACK(r13);                                    \
-        subi    r1,r1,SPECIAL_EXC_FRAME_SIZE;                               \
-        EXC_LEVEL_EXCEPTION_PROLOG(CRIT);
+        subi    r1,r1,SPECIAL_EXC_FRAME_SIZE
 #define SPRN_CRIT_SRR0  SPRN_CSRR0
 #define SPRN_CRIT_SRR1  SPRN_CSRR1
 
 #define DBG_SET_KSTACK                                                      \
         ld      r1,PACA_DBG_STACK(r13);                                     \
-        subi    r1,r1,SPECIAL_EXC_FRAME_SIZE;                               \
-        EXC_LEVEL_EXCEPTION_PROLOG(DBG);
+        subi    r1,r1,SPECIAL_EXC_FRAME_SIZE
 #define SPRN_DBG_SRR0   SPRN_DSRR0
 #define SPRN_DBG_SRR1   SPRN_DSRR1
 
 #define MC_SET_KSTACK                                                       \
         ld      r1,PACA_MC_STACK(r13);                                      \
-        subi    r1,r1,SPECIAL_EXC_FRAME_SIZE;                               \
-        EXC_LEVEL_EXCEPTION_PROLOG(MC);
+        subi    r1,r1,SPECIAL_EXC_FRAME_SIZE
 #define SPRN_MC_SRR0    SPRN_MCSRR0
 #define SPRN_MC_SRR1    SPRN_MCSRR1
 
@@ -322,27 +549,25 @@ interrupt_end_book3e:
         START_EXCEPTION(critical_input);
         CRIT_EXCEPTION_PROLOG(0x100, BOOKE_INTERRUPT_CRITICAL,
                               PROLOG_ADDITION_NONE)
-//      EXCEPTION_COMMON_CRIT(0x100)
-//      INTS_DISABLE
-//      bl      special_reg_save_crit
-//      CHECK_NAPPING();
-//      addi    r3,r1,STACK_FRAME_OVERHEAD
-//      bl      .critical_exception
-//      b       ret_from_crit_except
-        b       .
+        EXCEPTION_COMMON_CRIT(0x100)
+        bl      .save_nvgprs
+        bl      special_reg_save
+        CHECK_NAPPING();
+        addi    r3,r1,STACK_FRAME_OVERHEAD
+        bl      .unknown_exception
+        b       ret_from_crit_except
 
 /* Machine Check Interrupt */
         START_EXCEPTION(machine_check);
         MC_EXCEPTION_PROLOG(0x000, BOOKE_INTERRUPT_MACHINE_CHECK,
                             PROLOG_ADDITION_NONE)
-//      EXCEPTION_COMMON_MC(0x000)
-//      INTS_DISABLE
-//      bl      special_reg_save_mc
-//      addi    r3,r1,STACK_FRAME_OVERHEAD
-//      CHECK_NAPPING();
-//      bl      .machine_check_exception
-//      b       ret_from_mc_except
-        b       .
+        EXCEPTION_COMMON_MC(0x000)
+        bl      .save_nvgprs
+        bl      special_reg_save
+        CHECK_NAPPING();
+        addi    r3,r1,STACK_FRAME_OVERHEAD
+        bl      .machine_check_exception
+        b       ret_from_mc_except
 
 /* Data Storage Interrupt */
         START_EXCEPTION(data_storage)
@@ -461,14 +686,17 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
         START_EXCEPTION(watchdog);
         CRIT_EXCEPTION_PROLOG(0x9f0, BOOKE_INTERRUPT_WATCHDOG,
                               PROLOG_ADDITION_NONE)
-//      EXCEPTION_COMMON_CRIT(0x9f0)
-//      INTS_DISABLE
-//      bl      special_reg_save_crit
-//      CHECK_NAPPING();
-//      addi    r3,r1,STACK_FRAME_OVERHEAD
-//      bl      .unknown_exception
-//      b       ret_from_crit_except
-        b       .
+        EXCEPTION_COMMON_CRIT(0x9f0)
+        bl      .save_nvgprs
+        bl      special_reg_save
+        CHECK_NAPPING();
+        addi    r3,r1,STACK_FRAME_OVERHEAD
+#ifdef CONFIG_BOOKE_WDT
+        bl      .WatchdogException
+#else
+        bl      .unknown_exception
+#endif
+        b       ret_from_crit_except
 
 /* System Call Interrupt */
         START_EXCEPTION(system_call)
@@ -541,7 +769,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
          */
         mfspr   r14,SPRN_DBSR
         EXCEPTION_COMMON_CRIT(0xd00)
-        INTS_DISABLE
         std     r14,_DSISR(r1)
         addi    r3,r1,STACK_FRAME_OVERHEAD
         mr      r4,r14
@@ -634,14 +861,13 @@ kernel_dbg_exc:
         START_EXCEPTION(doorbell_crit);
         CRIT_EXCEPTION_PROLOG(0x2a0, BOOKE_INTERRUPT_DOORBELL_CRITICAL,
                               PROLOG_ADDITION_NONE)
-//      EXCEPTION_COMMON_CRIT(0x2a0)
-//      INTS_DISABLE
-//      bl      special_reg_save_crit
-//      CHECK_NAPPING();
-//      addi    r3,r1,STACK_FRAME_OVERHEAD
-//      bl      .doorbell_critical_exception
-//      b       ret_from_crit_except
-        b       .
+        EXCEPTION_COMMON_CRIT(0x2a0)
+        bl      .save_nvgprs
+        bl      special_reg_save
+        CHECK_NAPPING();
+        addi    r3,r1,STACK_FRAME_OVERHEAD
+        bl      .unknown_exception
+        b       ret_from_crit_except
 
 /*
  *      Guest doorbell interrupt
@@ -661,14 +887,13 @@ kernel_dbg_exc:
         START_EXCEPTION(guest_doorbell_crit);
         CRIT_EXCEPTION_PROLOG(0x2e0, BOOKE_INTERRUPT_GUEST_DBELL_CRIT,
                               PROLOG_ADDITION_NONE)
-//      EXCEPTION_COMMON_CRIT(0x2e0)
-//      INTS_DISABLE
-//      bl      special_reg_save_crit
-//      CHECK_NAPPING();
-//      addi    r3,r1,STACK_FRAME_OVERHEAD
-//      bl      .guest_doorbell_critical_exception
-//      b       ret_from_crit_except
-        b       .
+        EXCEPTION_COMMON_CRIT(0x2e0)
+        bl      .save_nvgprs
+        bl      special_reg_save
+        CHECK_NAPPING();
+        addi    r3,r1,STACK_FRAME_OVERHEAD
+        bl      .unknown_exception
+        b       ret_from_crit_except
 
 /* Hypervisor call */
         START_EXCEPTION(hypercall);
diff --git a/arch/powerpc/mm/tlb_nohash.c b/arch/powerpc/mm/tlb_nohash.c
index b37a58e1c92d..ae3d5b799b90 100644
--- a/arch/powerpc/mm/tlb_nohash.c
+++ b/arch/powerpc/mm/tlb_nohash.c
@@ -144,6 +144,15 @@ int mmu_vmemmap_psize;		/* Page size used for the virtual mem map */
 int book3e_htw_mode;            /* HW tablewalk?  Value is PPC_HTW_* */
 unsigned long linear_map_top;   /* Top of linear mapping */
 
+
+/*
+ * Number of bytes to add to SPRN_SPRG_TLB_EXFRAME on crit/mcheck/debug
+ * exceptions.  This is used for bolted and e6500 TLB miss handlers which
+ * do not modify this SPRG in the TLB miss code; for other TLB miss handlers,
+ * this is set to zero.
+ */
+int extlb_level_exc;
+
 #endif /* CONFIG_PPC64 */
 
 #ifdef CONFIG_PPC_FSL_BOOK3E
@@ -559,6 +568,7 @@ static void setup_mmu_htw(void)
                 break;
 #ifdef CONFIG_PPC_FSL_BOOK3E
         case PPC_HTW_E6500:
+                extlb_level_exc = EX_TLB_SIZE;
                 patch_exception(0x1c0, exc_data_tlb_miss_e6500_book3e);
                 patch_exception(0x1e0, exc_instruction_tlb_miss_e6500_book3e);
                 break;
@@ -652,6 +662,7 @@ static void __early_init_mmu(int boot_cpu)
                 memblock_enforce_memory_limit(linear_map_top);
 
                 if (book3e_htw_mode == PPC_HTW_NONE) {
+                        extlb_level_exc = EX_TLB_SIZE;
                         patch_exception(0x1c0, exc_data_tlb_miss_bolted_book3e);
                         patch_exception(0x1e0,
                                 exc_instruction_tlb_miss_bolted_book3e);