1 files changed, 155 insertions, 81 deletions

diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S
index 429983c06f91..7215cc2495df 100644
--- a/arch/powerpc/kernel/exceptions-64e.S
+++ b/arch/powerpc/kernel/exceptions-64e.S
@@ -24,6 +24,7 @@
 #include <asm/ptrace.h>
 #include <asm/ppc-opcode.h>
 #include <asm/mmu.h>
+#include <asm/hw_irq.h>
 
 /* XXX This will ultimately add space for a special exception save
  *     structure used to save things like SRR0/SRR1, SPRGs, MAS, etc...
@@ -77,59 +78,55 @@
 #define SPRN_MC_SRR1    SPRN_MCSRR1
 
 #define NORMAL_EXCEPTION_PROLOG(n, addition)                                \
-        EXCEPTION_PROLOG(n, GEN, addition##_GEN)
+        EXCEPTION_PROLOG(n, GEN, addition##_GEN(n))
 
 #define CRIT_EXCEPTION_PROLOG(n, addition)                                  \
-        EXCEPTION_PROLOG(n, CRIT, addition##_CRIT)
+        EXCEPTION_PROLOG(n, CRIT, addition##_CRIT(n))
 
 #define DBG_EXCEPTION_PROLOG(n, addition)                                   \
-        EXCEPTION_PROLOG(n, DBG, addition##_DBG)
+        EXCEPTION_PROLOG(n, DBG, addition##_DBG(n))
 
 #define MC_EXCEPTION_PROLOG(n, addition)                                    \
-        EXCEPTION_PROLOG(n, MC, addition##_MC)
+        EXCEPTION_PROLOG(n, MC, addition##_MC(n))
 
 
 /* Variants of the "addition" argument for the prolog
  */
-#define PROLOG_ADDITION_NONE_GEN
-#define PROLOG_ADDITION_NONE_CRIT
-#define PROLOG_ADDITION_NONE_DBG
-#define PROLOG_ADDITION_NONE_MC
+#define PROLOG_ADDITION_NONE_GEN(n)
+#define PROLOG_ADDITION_NONE_CRIT(n)
+#define PROLOG_ADDITION_NONE_DBG(n)
+#define PROLOG_ADDITION_NONE_MC(n)
 
-#define PROLOG_ADDITION_MASKABLE_GEN                                        \
+#define PROLOG_ADDITION_MASKABLE_GEN(n)                                     \
         lbz     r11,PACASOFTIRQEN(r13); /* are irqs soft-disabled ? */      \
         cmpwi   cr0,r11,0;              /* yes -> go out of line */         \
-        beq     masked_interrupt_book3e;
+        beq     masked_interrupt_book3e_##n
 
-#define PROLOG_ADDITION_2REGS_GEN                                           \
+#define PROLOG_ADDITION_2REGS_GEN(n)                                        \
         std     r14,PACA_EXGEN+EX_R14(r13);                                 \
         std     r15,PACA_EXGEN+EX_R15(r13)
 
-#define PROLOG_ADDITION_1REG_GEN                                            \
+#define PROLOG_ADDITION_1REG_GEN(n)                                         \
         std     r14,PACA_EXGEN+EX_R14(r13);
 
-#define PROLOG_ADDITION_2REGS_CRIT                                          \
+#define PROLOG_ADDITION_2REGS_CRIT(n)                                       \
         std     r14,PACA_EXCRIT+EX_R14(r13);                                \
         std     r15,PACA_EXCRIT+EX_R15(r13)
 
-#define PROLOG_ADDITION_2REGS_DBG                                           \
+#define PROLOG_ADDITION_2REGS_DBG(n)                                        \
         std     r14,PACA_EXDBG+EX_R14(r13);                                 \
         std     r15,PACA_EXDBG+EX_R15(r13)
 
-#define PROLOG_ADDITION_2REGS_MC                                            \
+#define PROLOG_ADDITION_2REGS_MC(n)                                         \
         std     r14,PACA_EXMC+EX_R14(r13);                                  \
         std     r15,PACA_EXMC+EX_R15(r13)
 
-#define PROLOG_ADDITION_DOORBELL_GEN                                        \
-        lbz     r11,PACASOFTIRQEN(r13); /* are irqs soft-disabled ? */      \
-        cmpwi   cr0,r11,0;              /* yes -> go out of line */         \
-        beq     masked_doorbell_book3e
-
 
 /* Core exception code for all exceptions except TLB misses.
  * XXX: Needs to make SPRN_SPRG_GEN depend on exception type
  */
 #define EXCEPTION_COMMON(n, excf, ints)                                     \
+exc_##n##_common:                                                           \
         std     r0,GPR0(r1);            /* save r0 in stackframe */         \
         std     r2,GPR2(r1);            /* save r2 in stackframe */         \
         SAVE_4GPRS(3, r1);              /* save r3 - r6 in stackframe */    \
@@ -167,20 +164,21 @@
         std     r0,RESULT(r1);          /* clear regs->result */            \
         ints;
 
-/* Variants for the "ints" argument */
+/* Variants for the "ints" argument. This one does nothing when we want
+ * to keep interrupts in their original state
+ */
 #define INTS_KEEP
-#define INTS_DISABLE_SOFT                                                   \
-        stb     r0,PACASOFTIRQEN(r13);  /* mark interrupts soft-disabled */ \
-        TRACE_DISABLE_INTS;
-#define INTS_DISABLE_HARD                                                   \
-        stb     r0,PACAHARDIRQEN(r13); /* and hard disabled */
-#define INTS_DISABLE_ALL                                                    \
-        INTS_DISABLE_SOFT                                                   \
-        INTS_DISABLE_HARD
-
-/* This is called by exceptions that used INTS_KEEP (that is did not clear
- * neither soft nor hard IRQ indicators in the PACA. This will restore MSR:EE
- * to it's previous value
+
+/* This second version is meant for exceptions that don't immediately
+ * hard-enable. We set a bit in paca->irq_happened to ensure that
+ * a subsequent call to arch_local_irq_restore() will properly
+ * hard-enable and avoid the fast-path
+ */
+#define INTS_DISABLE    SOFT_DISABLE_INTS(r3,r4)
+
+/* This is called by exceptions that used INTS_KEEP (that did not touch
+ * irq indicators in the PACA). This will restore MSR:EE to it's previous
+ * value
  *
  * XXX In the long run, we may want to open-code it in order to separate the
  *     load from the wrtee, thus limiting the latency caused by the dependency
@@ -238,7 +236,7 @@ exc_##n##_bad_stack:							    \
 #define MASKABLE_EXCEPTION(trapnum, label, hdlr, ack)                   \
         START_EXCEPTION(label);                                         \
         NORMAL_EXCEPTION_PROLOG(trapnum, PROLOG_ADDITION_MASKABLE)      \
-        EXCEPTION_COMMON(trapnum, PACA_EXGEN, INTS_DISABLE_ALL)         \
+        EXCEPTION_COMMON(trapnum, PACA_EXGEN, INTS_DISABLE)             \
         ack(r8);                                                        \
         CHECK_NAPPING();                                                \
         addi    r3,r1,STACK_FRAME_OVERHEAD;                             \
@@ -289,7 +287,7 @@ interrupt_end_book3e:
 /* Critical Input Interrupt */
         START_EXCEPTION(critical_input);
         CRIT_EXCEPTION_PROLOG(0x100, PROLOG_ADDITION_NONE)
-//      EXCEPTION_COMMON(0x100, PACA_EXCRIT, INTS_DISABLE_ALL)
+//      EXCEPTION_COMMON(0x100, PACA_EXCRIT, INTS_DISABLE)
 //      bl      special_reg_save_crit
 //      CHECK_NAPPING();
 //      addi    r3,r1,STACK_FRAME_OVERHEAD
@@ -300,7 +298,7 @@ interrupt_end_book3e:
 /* Machine Check Interrupt */
         START_EXCEPTION(machine_check);
         CRIT_EXCEPTION_PROLOG(0x200, PROLOG_ADDITION_NONE)
-//      EXCEPTION_COMMON(0x200, PACA_EXMC, INTS_DISABLE_ALL)
+//      EXCEPTION_COMMON(0x200, PACA_EXMC, INTS_DISABLE)
 //      bl      special_reg_save_mc
 //      addi    r3,r1,STACK_FRAME_OVERHEAD
 //      CHECK_NAPPING();
@@ -313,7 +311,7 @@ interrupt_end_book3e:
         NORMAL_EXCEPTION_PROLOG(0x300, PROLOG_ADDITION_2REGS)
         mfspr   r14,SPRN_DEAR
         mfspr   r15,SPRN_ESR
-        EXCEPTION_COMMON(0x300, PACA_EXGEN, INTS_KEEP)
+        EXCEPTION_COMMON(0x300, PACA_EXGEN, INTS_DISABLE)
         b       storage_fault_common
 
 /* Instruction Storage Interrupt */
@@ -321,7 +319,7 @@ interrupt_end_book3e:
         NORMAL_EXCEPTION_PROLOG(0x400, PROLOG_ADDITION_2REGS)
         li      r15,0
         mr      r14,r10
-        EXCEPTION_COMMON(0x400, PACA_EXGEN, INTS_KEEP)
+        EXCEPTION_COMMON(0x400, PACA_EXGEN, INTS_DISABLE)
         b       storage_fault_common
 
 /* External Input Interrupt */
@@ -339,12 +337,11 @@ interrupt_end_book3e:
         START_EXCEPTION(program);
         NORMAL_EXCEPTION_PROLOG(0x700, PROLOG_ADDITION_1REG)
         mfspr   r14,SPRN_ESR
-        EXCEPTION_COMMON(0x700, PACA_EXGEN, INTS_DISABLE_SOFT)
+        EXCEPTION_COMMON(0x700, PACA_EXGEN, INTS_DISABLE)
         std     r14,_DSISR(r1)
         addi    r3,r1,STACK_FRAME_OVERHEAD
         ld      r14,PACA_EXGEN+EX_R14(r13)
         bl      .save_nvgprs
-        INTS_RESTORE_HARD
         bl      .program_check_exception
         b       .ret_from_except
 
@@ -353,15 +350,16 @@ interrupt_end_book3e:
         NORMAL_EXCEPTION_PROLOG(0x800, PROLOG_ADDITION_NONE)
         /* we can probably do a shorter exception entry for that one... */
         EXCEPTION_COMMON(0x800, PACA_EXGEN, INTS_KEEP)
-        bne     1f                      /* if from user, just load it up */
+        ld      r12,_MSR(r1)
+        andi.   r0,r12,MSR_PR;
+        beq-    1f
+        bl      .load_up_fpu
+        b       fast_exception_return
+1:      INTS_DISABLE
         bl      .save_nvgprs
         addi    r3,r1,STACK_FRAME_OVERHEAD
-        INTS_RESTORE_HARD
         bl      .kernel_fp_unavailable_exception
-        BUG_OPCODE
-1:      ld      r12,_MSR(r1)
-        bl      .load_up_fpu
-        b       fast_exception_return
+        b       .ret_from_except
 
 /* Decrementer Interrupt */
         MASKABLE_EXCEPTION(0x900, decrementer, .timer_interrupt, ACK_DEC)
@@ -372,7 +370,7 @@ interrupt_end_book3e:
 /* Watchdog Timer Interrupt */
         START_EXCEPTION(watchdog);
         CRIT_EXCEPTION_PROLOG(0x9f0, PROLOG_ADDITION_NONE)
-//      EXCEPTION_COMMON(0x9f0, PACA_EXCRIT, INTS_DISABLE_ALL)
+//      EXCEPTION_COMMON(0x9f0, PACA_EXCRIT, INTS_DISABLE)
 //      bl      special_reg_save_crit
 //      CHECK_NAPPING();
 //      addi    r3,r1,STACK_FRAME_OVERHEAD
@@ -391,10 +389,9 @@ interrupt_end_book3e:
 /* Auxiliary Processor Unavailable Interrupt */
         START_EXCEPTION(ap_unavailable);
         NORMAL_EXCEPTION_PROLOG(0xf20, PROLOG_ADDITION_NONE)
-        EXCEPTION_COMMON(0xf20, PACA_EXGEN, INTS_KEEP)
-        addi    r3,r1,STACK_FRAME_OVERHEAD
+        EXCEPTION_COMMON(0xf20, PACA_EXGEN, INTS_DISABLE)
         bl      .save_nvgprs
-        INTS_RESTORE_HARD
+        addi    r3,r1,STACK_FRAME_OVERHEAD
         bl      .unknown_exception
         b       .ret_from_except
 
@@ -450,7 +447,7 @@ interrupt_end_book3e:
         mfspr   r15,SPRN_SPRG_CRIT_SCRATCH
         mtspr   SPRN_SPRG_GEN_SCRATCH,r15
         mfspr   r14,SPRN_DBSR
-        EXCEPTION_COMMON(0xd00, PACA_EXCRIT, INTS_DISABLE_ALL)
+        EXCEPTION_COMMON(0xd00, PACA_EXCRIT, INTS_DISABLE)
         std     r14,_DSISR(r1)
         addi    r3,r1,STACK_FRAME_OVERHEAD
         mr      r4,r14
@@ -465,7 +462,7 @@ kernel_dbg_exc:
 
 /* Debug exception as a debug interrupt*/
         START_EXCEPTION(debug_debug);
-        DBG_EXCEPTION_PROLOG(0xd00, PROLOG_ADDITION_2REGS)
+        DBG_EXCEPTION_PROLOG(0xd08, PROLOG_ADDITION_2REGS)
 
         /*
          * If there is a single step or branch-taken exception in an
@@ -515,7 +512,7 @@ kernel_dbg_exc:
         mfspr   r15,SPRN_SPRG_DBG_SCRATCH
         mtspr   SPRN_SPRG_GEN_SCRATCH,r15
         mfspr   r14,SPRN_DBSR
-        EXCEPTION_COMMON(0xd00, PACA_EXDBG, INTS_DISABLE_ALL)
+        EXCEPTION_COMMON(0xd08, PACA_EXDBG, INTS_DISABLE)
         std     r14,_DSISR(r1)
         addi    r3,r1,STACK_FRAME_OVERHEAD
         mr      r4,r14
@@ -525,21 +522,20 @@ kernel_dbg_exc:
         bl      .DebugException
         b       .ret_from_except
 
-        MASKABLE_EXCEPTION(0x260, perfmon, .performance_monitor_exception, ACK_NONE)
-
-/* Doorbell interrupt */
-        START_EXCEPTION(doorbell)
-        NORMAL_EXCEPTION_PROLOG(0x2070, PROLOG_ADDITION_DOORBELL)
-        EXCEPTION_COMMON(0x2070, PACA_EXGEN, INTS_DISABLE_ALL)
-        CHECK_NAPPING()
+        START_EXCEPTION(perfmon);
+        NORMAL_EXCEPTION_PROLOG(0x260, PROLOG_ADDITION_NONE)
+        EXCEPTION_COMMON(0x260, PACA_EXGEN, INTS_DISABLE)
         addi    r3,r1,STACK_FRAME_OVERHEAD
-        bl      .doorbell_exception
+        bl      .performance_monitor_exception
         b       .ret_from_except_lite
 
+/* Doorbell interrupt */
+        MASKABLE_EXCEPTION(0x280, doorbell, .doorbell_exception, ACK_NONE)
+
 /* Doorbell critical Interrupt */
         START_EXCEPTION(doorbell_crit);
-        CRIT_EXCEPTION_PROLOG(0x2080, PROLOG_ADDITION_NONE)
-//      EXCEPTION_COMMON(0x2080, PACA_EXCRIT, INTS_DISABLE_ALL)
+        CRIT_EXCEPTION_PROLOG(0x2a0, PROLOG_ADDITION_NONE)
+//      EXCEPTION_COMMON(0x2a0, PACA_EXCRIT, INTS_DISABLE)
 //      bl      special_reg_save_crit
 //      CHECK_NAPPING();
 //      addi    r3,r1,STACK_FRAME_OVERHEAD
@@ -547,36 +543,114 @@ kernel_dbg_exc:
 //      b       ret_from_crit_except
         b       .
 
+/* Guest Doorbell */
         MASKABLE_EXCEPTION(0x2c0, guest_doorbell, .unknown_exception, ACK_NONE)
-        MASKABLE_EXCEPTION(0x2e0, guest_doorbell_crit, .unknown_exception, ACK_NONE)
-        MASKABLE_EXCEPTION(0x310, hypercall, .unknown_exception, ACK_NONE)
-        MASKABLE_EXCEPTION(0x320, ehpriv, .unknown_exception, ACK_NONE)
 
+/* Guest Doorbell critical Interrupt */
+        START_EXCEPTION(guest_doorbell_crit);
+        CRIT_EXCEPTION_PROLOG(0x2e0, PROLOG_ADDITION_NONE)
+//      EXCEPTION_COMMON(0x2e0, PACA_EXCRIT, INTS_DISABLE)
+//      bl      special_reg_save_crit
+//      CHECK_NAPPING();
+//      addi    r3,r1,STACK_FRAME_OVERHEAD
+//      bl      .guest_doorbell_critical_exception
+//      b       ret_from_crit_except
+        b       .
+
+/* Hypervisor call */
+        START_EXCEPTION(hypercall);
+        NORMAL_EXCEPTION_PROLOG(0x310, PROLOG_ADDITION_NONE)
+        EXCEPTION_COMMON(0x310, PACA_EXGEN, INTS_KEEP)
+        addi    r3,r1,STACK_FRAME_OVERHEAD
+        bl      .save_nvgprs
+        INTS_RESTORE_HARD
+        bl      .unknown_exception
+        b       .ret_from_except
+
+/* Embedded Hypervisor priviledged  */
+        START_EXCEPTION(ehpriv);
+        NORMAL_EXCEPTION_PROLOG(0x320, PROLOG_ADDITION_NONE)
+        EXCEPTION_COMMON(0x320, PACA_EXGEN, INTS_KEEP)
+        addi    r3,r1,STACK_FRAME_OVERHEAD
+        bl      .save_nvgprs
+        INTS_RESTORE_HARD
+        bl      .unknown_exception
+        b       .ret_from_except
 
 /*
- * An interrupt came in while soft-disabled; clear EE in SRR1,
- * clear paca->hard_enabled and return.
+ * An interrupt came in while soft-disabled; We mark paca->irq_happened
+ * accordingly and if the interrupt is level sensitive, we hard disable
  */
-masked_doorbell_book3e:
-        mtcr    r10
-        /* Resend the doorbell to fire again when ints enabled */
-        mfspr   r10,SPRN_PIR
-        PPC_MSGSND(r10)
-        b       masked_interrupt_book3e_common
 
-masked_interrupt_book3e:
+masked_interrupt_book3e_0x500:
+        /* XXX When adding support for EPR, use PACA_IRQ_EE_EDGE */
+        li      r11,PACA_IRQ_EE
+        b       masked_interrupt_book3e_full_mask
+
+masked_interrupt_book3e_0x900:
+        ACK_DEC(r11);
+        li      r11,PACA_IRQ_DEC
+        b       masked_interrupt_book3e_no_mask
+masked_interrupt_book3e_0x980:
+        ACK_FIT(r11);
+        li      r11,PACA_IRQ_DEC
+        b       masked_interrupt_book3e_no_mask
+masked_interrupt_book3e_0x280:
+masked_interrupt_book3e_0x2c0:
+        li      r11,PACA_IRQ_DBELL
+        b       masked_interrupt_book3e_no_mask
+
+masked_interrupt_book3e_no_mask:
+        mtcr    r10
+        lbz     r10,PACAIRQHAPPENED(r13)
+        or      r10,r10,r11
+        stb     r10,PACAIRQHAPPENED(r13)
+        b       1f
+masked_interrupt_book3e_full_mask:
         mtcr    r10
-masked_interrupt_book3e_common:
-        stb     r11,PACAHARDIRQEN(r13)
+        lbz     r10,PACAIRQHAPPENED(r13)
+        or      r10,r10,r11
+        stb     r10,PACAIRQHAPPENED(r13)
         mfspr   r10,SPRN_SRR1
         rldicl  r11,r10,48,1            /* clear MSR_EE */
         rotldi  r10,r11,16
         mtspr   SPRN_SRR1,r10
-        ld      r10,PACA_EXGEN+EX_R10(r13);     /* restore registers */
+1:      ld      r10,PACA_EXGEN+EX_R10(r13);
         ld      r11,PACA_EXGEN+EX_R11(r13);
         mfspr   r13,SPRN_SPRG_GEN_SCRATCH;
         rfi
         b       .
+/*
+ * Called from arch_local_irq_enable when an interrupt needs
+ * to be resent. r3 contains either 0x500,0x900,0x260 or 0x280
+ * to indicate the kind of interrupt. MSR:EE is already off.
+ * We generate a stackframe like if a real interrupt had happened.
+ *
+ * Note: While MSR:EE is off, we need to make sure that _MSR
+ * in the generated frame has EE set to 1 or the exception
+ * handler will not properly re-enable them.
+ */
+_GLOBAL(__replay_interrupt)
+        /* We are going to jump to the exception common code which
+         * will retrieve various register values from the PACA which
+         * we don't give a damn about.
+         */
+        mflr    r10
+        mfmsr   r11
+        mfcr    r4
+        mtspr   SPRN_SPRG_GEN_SCRATCH,r13;
+        std     r1,PACA_EXGEN+EX_R1(r13);
+        stw     r4,PACA_EXGEN+EX_CR(r13);
+        ori     r11,r11,MSR_EE
+        subi    r1,r1,INT_FRAME_SIZE;
+        cmpwi   cr0,r3,0x500
+        beq     exc_0x500_common
+        cmpwi   cr0,r3,0x900
+        beq     exc_0x900_common
+        cmpwi   cr0,r3,0x280
+        beq     exc_0x280_common
+        blr
+
 
 /*
  * This is called from 0x300 and 0x400 handlers after the prologs with
@@ -591,7 +665,6 @@ storage_fault_common:
         mr      r5,r15
         ld      r14,PACA_EXGEN+EX_R14(r13)
         ld      r15,PACA_EXGEN+EX_R15(r13)
-        INTS_RESTORE_HARD
         bl      .do_page_fault
         cmpdi   r3,0
         bne-    1f
@@ -680,6 +753,8 @@ BAD_STACK_TRAMPOLINE(0x000)
 BAD_STACK_TRAMPOLINE(0x100)
 BAD_STACK_TRAMPOLINE(0x200)
 BAD_STACK_TRAMPOLINE(0x260)
+BAD_STACK_TRAMPOLINE(0x280)
+BAD_STACK_TRAMPOLINE(0x2a0)
 BAD_STACK_TRAMPOLINE(0x2c0)
 BAD_STACK_TRAMPOLINE(0x2e0)
 BAD_STACK_TRAMPOLINE(0x300)
@@ -697,11 +772,10 @@ BAD_STACK_TRAMPOLINE(0xa00)
 BAD_STACK_TRAMPOLINE(0xb00)
 BAD_STACK_TRAMPOLINE(0xc00)
 BAD_STACK_TRAMPOLINE(0xd00)
+BAD_STACK_TRAMPOLINE(0xd08)
 BAD_STACK_TRAMPOLINE(0xe00)
 BAD_STACK_TRAMPOLINE(0xf00)
 BAD_STACK_TRAMPOLINE(0xf20)
-BAD_STACK_TRAMPOLINE(0x2070)
-BAD_STACK_TRAMPOLINE(0x2080)
 
         .globl  bad_stack_book3e
 bad_stack_book3e: