19 files changed, 647 insertions, 342 deletions

diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h
index 70354af0740e..548da3aa0a30 100644
--- a/arch/powerpc/include/asm/exception-64s.h
+++ b/arch/powerpc/include/asm/exception-64s.h
@@ -232,23 +232,30 @@ label##_hv:						\
         EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common,    \
                                  EXC_HV, KVMTEST, vec)
 
-#define __SOFTEN_TEST(h)                                                \
+/* This associate vector numbers with bits in paca->irq_happened */
+#define SOFTEN_VALUE_0x500      PACA_IRQ_EE
+#define SOFTEN_VALUE_0x502      PACA_IRQ_EE
+#define SOFTEN_VALUE_0x900      PACA_IRQ_DEC
+#define SOFTEN_VALUE_0x982      PACA_IRQ_DEC
+
+#define __SOFTEN_TEST(h, vec)                                           \
         lbz     r10,PACASOFTIRQEN(r13);                                 \
         cmpwi   r10,0;                                                  \
+        li      r10,SOFTEN_VALUE_##vec;                                 \
         beq     masked_##h##interrupt
-#define _SOFTEN_TEST(h) __SOFTEN_TEST(h)
+#define _SOFTEN_TEST(h, vec)    __SOFTEN_TEST(h, vec)
 
 #define SOFTEN_TEST_PR(vec)                                             \
         KVMTEST_PR(vec);                                                \
-        _SOFTEN_TEST(EXC_STD)
+        _SOFTEN_TEST(EXC_STD, vec)
 
 #define SOFTEN_TEST_HV(vec)                                             \
         KVMTEST(vec);                                                   \
-        _SOFTEN_TEST(EXC_HV)
+        _SOFTEN_TEST(EXC_HV, vec)
 
 #define SOFTEN_TEST_HV_201(vec)                                         \
         KVMTEST(vec);                                                   \
-        _SOFTEN_TEST(EXC_STD)
+        _SOFTEN_TEST(EXC_STD, vec)
 
 #define __MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra)              \
         HMT_MEDIUM;                                                     \
@@ -279,22 +286,7 @@ label##_hv:								\
  */
 
 /* Exception addition: Hard disable interrupts */
-#ifdef CONFIG_TRACE_IRQFLAGS
-#define DISABLE_INTS                            \
-        lbz     r10,PACASOFTIRQEN(r13);         \
-        li      r11,0;                          \
-        cmpwi   cr0,r10,0;                      \
-        stb     r11,PACAHARDIRQEN(r13);         \
-        beq     44f;                            \
-        stb     r11,PACASOFTIRQEN(r13);         \
-        TRACE_DISABLE_INTS;                     \
-44:
-#else
-#define DISABLE_INTS                            \
-        li      r11,0;                          \
-        stb     r11,PACASOFTIRQEN(r13);         \
-        stb     r11,PACAHARDIRQEN(r13)
-#endif /* CONFIG_TRACE_IRQFLAGS */
+#define DISABLE_INTS    SOFT_DISABLE_INTS(r10,r11)
 
 /* Exception addition: Keep interrupt state */
 #define ENABLE_INTS                             \
diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h
index 6c6fa955baa7..51010bfc792e 100644
--- a/arch/powerpc/include/asm/hw_irq.h
+++ b/arch/powerpc/include/asm/hw_irq.h
@@ -11,6 +11,27 @@
 #include <asm/ptrace.h>
 #include <asm/processor.h>
 
+#ifdef CONFIG_PPC64
+
+/*
+ * PACA flags in paca->irq_happened.
+ *
+ * This bits are set when interrupts occur while soft-disabled
+ * and allow a proper replay. Additionally, PACA_IRQ_HARD_DIS
+ * is set whenever we manually hard disable.
+ */
+#define PACA_IRQ_HARD_DIS       0x01
+#define PACA_IRQ_DBELL          0x02
+#define PACA_IRQ_EE             0x04
+#define PACA_IRQ_DEC            0x08 /* Or FIT */
+#define PACA_IRQ_EE_EDGE        0x10 /* BookE only */
+
+#endif /* CONFIG_PPC64 */
+
+#ifndef __ASSEMBLY__
+
+extern void __replay_interrupt(unsigned int vector);
+
 extern void timer_interrupt(struct pt_regs *);
 
 #ifdef CONFIG_PPC64
@@ -42,7 +63,6 @@ static inline unsigned long arch_local_irq_disable(void)
 }
 
 extern void arch_local_irq_restore(unsigned long);
-extern void iseries_handle_interrupts(void);
 
 static inline void arch_local_irq_enable(void)
 {
@@ -72,12 +92,24 @@ static inline bool arch_irqs_disabled(void)
 #define __hard_irq_disable()    __mtmsrd(local_paca->kernel_msr, 1)
 #endif
 
-#define  hard_irq_disable()                     \
-        do {                                    \
-                __hard_irq_disable();           \
-                get_paca()->soft_enabled = 0;   \
-                get_paca()->hard_enabled = 0;   \
-        } while(0)
+static inline void hard_irq_disable(void)
+{
+        __hard_irq_disable();
+        get_paca()->soft_enabled = 0;
+        get_paca()->irq_happened |= PACA_IRQ_HARD_DIS;
+}
+
+/*
+ * This is called by asynchronous interrupts to conditionally
+ * re-enable hard interrupts when soft-disabled after having
+ * cleared the source of the interrupt
+ */
+static inline void may_hard_irq_enable(void)
+{
+        get_paca()->irq_happened &= ~PACA_IRQ_HARD_DIS;
+        if (!(get_paca()->irq_happened & PACA_IRQ_EE))
+                __hard_irq_enable();
+}
 
 static inline bool arch_irq_disabled_regs(struct pt_regs *regs)
 {
@@ -149,6 +181,8 @@ static inline bool arch_irq_disabled_regs(struct pt_regs *regs)
         return !(regs->msr & MSR_EE);
 }
 
+static inline void may_hard_irq_enable(void) { }
+
 #endif /* CONFIG_PPC64 */
 
 #define ARCH_IRQ_INIT_FLAGS     IRQ_NOREQUEST
@@ -159,5 +193,6 @@ static inline bool arch_irq_disabled_regs(struct pt_regs *regs)
  */
 struct irq_chip;
 
+#endif  /* __ASSEMBLY__ */
 #endif  /* __KERNEL__ */
 #endif  /* _ASM_POWERPC_HW_IRQ_H */
diff --git a/arch/powerpc/include/asm/irqflags.h b/arch/powerpc/include/asm/irqflags.h
index b0b06d85788d..6f9b6e23dc5a 100644
--- a/arch/powerpc/include/asm/irqflags.h
+++ b/arch/powerpc/include/asm/irqflags.h
@@ -39,24 +39,31 @@
 #define TRACE_ENABLE_INTS       TRACE_WITH_FRAME_BUFFER(.trace_hardirqs_on)
 #define TRACE_DISABLE_INTS      TRACE_WITH_FRAME_BUFFER(.trace_hardirqs_off)
 
-#define TRACE_AND_RESTORE_IRQ_PARTIAL(en,skip)          \
-        cmpdi   en,0;                                   \
-        bne     95f;                                    \
-        stb     en,PACASOFTIRQEN(r13);                  \
-        TRACE_WITH_FRAME_BUFFER(.trace_hardirqs_off)    \
-        b       skip;                                   \
-95:     TRACE_WITH_FRAME_BUFFER(.trace_hardirqs_on)     \
-        li      en,1;
-#define TRACE_AND_RESTORE_IRQ(en)               \
-        TRACE_AND_RESTORE_IRQ_PARTIAL(en,96f);  \
-        stb     en,PACASOFTIRQEN(r13);          \
-96:
+/*
+ * This is used by assembly code to soft-disable interrupts
+ */
+#define SOFT_DISABLE_INTS(__rA, __rB)           \
+        lbz     __rA,PACASOFTIRQEN(r13);        \
+        lbz     __rB,PACAIRQHAPPENED(r13);      \
+        cmpwi   cr0,__rA,0;                     \
+        li      __rA,0;                         \
+        ori     __rB,__rB,PACA_IRQ_HARD_DIS;    \
+        stb     __rB,PACAIRQHAPPENED(r13);      \
+        beq     44f;                            \
+        stb     __rA,PACASOFTIRQEN(r13);        \
+        TRACE_DISABLE_INTS;                     \
+44:
+
 #else
 #define TRACE_ENABLE_INTS
 #define TRACE_DISABLE_INTS
-#define TRACE_AND_RESTORE_IRQ_PARTIAL(en,skip)
-#define TRACE_AND_RESTORE_IRQ(en)               \
-        stb     en,PACASOFTIRQEN(r13)
+
+#define SOFT_DISABLE_INTS(__rA, __rB)           \
+        lbz     __rA,PACAIRQHAPPENED(r13);      \
+        li      __rB,0;                         \
+        ori     __rA,__rA,PACA_IRQ_HARD_DIS;    \
+        stb     __rB,PACASOFTIRQEN(r13);        \
+        stb     __rA,PACAIRQHAPPENED(r13)
 #endif
 #endif
 
diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h
index 269c05a36d91..daf813fea91f 100644
--- a/arch/powerpc/include/asm/paca.h
+++ b/arch/powerpc/include/asm/paca.h
@@ -132,7 +132,7 @@ struct paca_struct {
         u64 saved_msr;                  /* MSR saved here by enter_rtas */
         u16 trap_save;                  /* Used when bad stack is encountered */
         u8 soft_enabled;                /* irq soft-enable flag */
-        u8 hard_enabled;                /* set if irqs are enabled in MSR */
+        u8 irq_happened;                /* irq happened while soft-disabled */
         u8 io_sync;                     /* writel() needs spin_unlock sync */
         u8 irq_work_pending;            /* IRQ_WORK interrupt while soft-disable */
         u8 nap_state_lost;              /* NV GPR values lost in power7_idle */
diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index 04caee7d9bc1..cdd0d264415f 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -147,7 +147,7 @@ int main(void)
         DEFINE(PACAKBASE, offsetof(struct paca_struct, kernelbase));
         DEFINE(PACAKMSR, offsetof(struct paca_struct, kernel_msr));
         DEFINE(PACASOFTIRQEN, offsetof(struct paca_struct, soft_enabled));
-        DEFINE(PACAHARDIRQEN, offsetof(struct paca_struct, hard_enabled));
+        DEFINE(PACAIRQHAPPENED, offsetof(struct paca_struct, irq_happened));
         DEFINE(PACACONTEXTID, offsetof(struct paca_struct, context.id));
 #ifdef CONFIG_PPC_MM_SLICES
         DEFINE(PACALOWSLICESPSIZE, offsetof(struct paca_struct,
diff --git a/arch/powerpc/kernel/dbell.c b/arch/powerpc/kernel/dbell.c
index 2cc451aaaca7..5b25c8060fd6 100644
--- a/arch/powerpc/kernel/dbell.c
+++ b/arch/powerpc/kernel/dbell.c
@@ -37,6 +37,8 @@ void doorbell_exception(struct pt_regs *regs)
 
         irq_enter();
 
+        may_hard_irq_enable();
+
         smp_ipi_demux();
 
         irq_exit();
diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S
index c513beb78b3b..f8a7a1a1a9f4 100644
--- a/arch/powerpc/kernel/entry_64.S
+++ b/arch/powerpc/kernel/entry_64.S
@@ -32,6 +32,7 @@
 #include <asm/ptrace.h>
 #include <asm/irqflags.h>
 #include <asm/ftrace.h>
+#include <asm/hw_irq.h>
 
 /*
  * System calls.
@@ -583,18 +584,72 @@ _GLOBAL(ret_from_except_lite)
         bne     do_work
 #endif /* !CONFIG_PREEMPT */
 
+        .globl  fast_exc_return_irq
+fast_exc_return_irq:
 restore:
+        /*
+         * This is the main kernel exit path, we first check if we
+         * have to change our interrupt state.
+         */
         ld      r5,SOFTE(r1)
-        TRACE_AND_RESTORE_IRQ(r5);
+        lbz     r6,PACASOFTIRQEN(r13)
+        cmpwi   cr1,r5,0
+        cmpw    cr0,r5,r6
+        beq     cr0,4f
+
+        /* We do, handle disable first, which is easy */
+        bne     cr1,3f;
+        li      r0,0
+        stb     r0,PACASOFTIRQEN(r13);
+        TRACE_DISABLE_INTS
+        b       4f
 
-        /* extract EE bit and use it to restore paca->hard_enabled */
-        ld      r3,_MSR(r1)
-        rldicl  r4,r3,49,63             /* r0 = (r3 >> 15) & 1 */
-        stb     r4,PACAHARDIRQEN(r13)
+3:      /*
+         * We are about to soft-enable interrupts (we are hard disabled
+         * at this point). We check if there's anything that needs to
+         * be replayed first.
+         */
+        lbz     r0,PACAIRQHAPPENED(r13)
+        cmpwi   cr0,r0,0
+        bne-    restore_check_irq_replay
+
+        /*
+         * Get here when nothing happened while soft-disabled, just
+         * soft-enable and move-on. We will hard-enable as a side
+         * effect of rfi
+         */
+restore_no_replay:
+        TRACE_ENABLE_INTS
+        li      r0,1
+        stb     r0,PACASOFTIRQEN(r13);
 
+        /*
+         * Final return path. BookE is handled in a different file
+         */
+4:
 #ifdef CONFIG_PPC_BOOK3E
         b       .exception_return_book3e
 #else
+        /*
+         * Clear the reservation. If we know the CPU tracks the address of
+         * the reservation then we can potentially save some cycles and use
+         * a larx. On POWER6 and POWER7 this is significantly faster.
+         */
+BEGIN_FTR_SECTION
+        stdcx.  r0,0,r1         /* to clear the reservation */
+FTR_SECTION_ELSE
+        ldarx   r4,0,r1
+ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
+
+        /*
+         * Some code path such as load_up_fpu or altivec return directly
+         * here. They run entirely hard disabled and do not alter the
+         * interrupt state. They also don't use lwarx/stwcx. and thus
+         * are known not to leave dangling reservations.
+         */
+        .globl  fast_exception_return
+fast_exception_return:
+        ld      r3,_MSR(r1)
         ld      r4,_CTR(r1)
         ld      r0,_LINK(r1)
         mtctr   r4
@@ -608,17 +663,6 @@ restore:
         beq-    unrecov_restore
 
         /*
-         * Clear the reservation. If we know the CPU tracks the address of
-         * the reservation then we can potentially save some cycles and use
-         * a larx. On POWER6 and POWER7 this is significantly faster.
-         */
-BEGIN_FTR_SECTION
-        stdcx.  r0,0,r1         /* to clear the reservation */
-FTR_SECTION_ELSE
-        ldarx   r4,0,r1
-ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
-
-        /*
          * Clear RI before restoring r13.  If we are returning to
          * userspace and we take an exception after restoring r13,
          * we end up corrupting the userspace r13 value.
@@ -629,7 +673,8 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
 
         /*
          * r13 is our per cpu area, only restore it if we are returning to
-         * userspace
+         * userspace the value stored in the stack frame may belong to
+         * another CPU.
          */
         andi.   r0,r3,MSR_PR
         beq     1f
@@ -654,6 +699,55 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
 
 #endif /* CONFIG_PPC_BOOK3E */
 
+        /*
+         * Something did happen, check if a re-emit is needed
+         * (this also clears paca->irq_happened)
+         */
+restore_check_irq_replay:
+        /* XXX: We could implement a fast path here where we check
+         * for irq_happened being just 0x01, in which case we can
+         * clear it and return. That means that we would potentially
+         * miss a decrementer having wrapped all the way around.
+         *
+         * Still, this might be useful for things like hash_page
+         */
+        bl      .__check_irq_replay
+        cmpwi   cr0,r3,0
+        beq     restore_no_replay
+ 
+        /*
+         * We need to re-emit an interrupt. We do so by re-using our
+         * existing exception frame. We first change the trap value,
+         * but we need to ensure we preserve the low nibble of it
+         */
+        ld      r4,_TRAP(r1)
+        clrldi  r4,r4,60
+        or      r4,r4,r3
+        std     r4,_TRAP(r1)
+
+        /*
+         * Then find the right handler and call it. Interrupts are
+         * still soft-disabled and we keep them that way.
+        */
+        cmpwi   cr0,r3,0x500
+        bne     1f
+        addi    r3,r1,STACK_FRAME_OVERHEAD;
+        bl      .do_IRQ
+        b       .ret_from_except
+1:      cmpwi   cr0,r3,0x900
+        bne     1f
+        addi    r3,r1,STACK_FRAME_OVERHEAD;
+        bl      .timer_interrupt
+        b       .ret_from_except
+#ifdef CONFIG_PPC_BOOK3E
+1:      cmpwi   cr0,r3,0x280
+        bne     1f
+        addi    r3,r1,STACK_FRAME_OVERHEAD;
+        bl      .doorbell_exception
+        b       .ret_from_except
+#endif /* CONFIG_PPC_BOOK3E */
+1:      b       .ret_from_except /* What else to do here ? */
+ 
 do_work:
 #ifdef CONFIG_PREEMPT
         andi.   r0,r3,MSR_PR    /* Returning to user mode? */
@@ -666,18 +760,11 @@ do_work:
         crandc  eq,cr1*4+eq,eq
         bne     restore
 
-        /* Here we are preempting the current task.
-         *
-         * Ensure interrupts are soft-disabled. We also properly mark
-         * the PACA to reflect the fact that they are hard-disabled
-         * and trace the change
+        /*
+         * Here we are preempting the current task. We want to make
+         * sure we are soft-disabled first
          */
-        li      r0,0
-        stb     r0,PACASOFTIRQEN(r13)
-        stb     r0,PACAHARDIRQEN(r13)
-        TRACE_DISABLE_INTS
-
-        /* Call the scheduler with soft IRQs off */
+        SOFT_DISABLE_INTS(r3,r4)
 1:      bl      .preempt_schedule_irq
 
         /* Hard-disable interrupts again (and update PACA) */
@@ -687,8 +774,8 @@ do_work:
         ld      r10,PACAKMSR(r13) /* Get kernel MSR without EE */
         mtmsrd  r10,1
 #endif /* CONFIG_PPC_BOOK3E */
-        li      r0,0
-        stb     r0,PACAHARDIRQEN(r13)
+        li      r0,PACA_IRQ_HARD_DIS
+        stb     r0,PACAIRQHAPPENED(r13)
 
         /* Re-test flags and eventually loop */
         clrrdi  r9,r1,THREAD_SHIFT
@@ -710,14 +797,12 @@ user_work:
 
         andi.   r0,r4,_TIF_NEED_RESCHED
         beq     1f
-        li      r5,1
-        TRACE_AND_RESTORE_IRQ(r5);
+        bl      .restore_interrupts
         bl      .schedule
         b       .ret_from_except_lite
 
 1:      bl      .save_nvgprs
-        li      r5,1
-        TRACE_AND_RESTORE_IRQ(r5);
+        bl      .restore_interrupts
         addi    r3,r1,STACK_FRAME_OVERHEAD
         bl      .do_notify_resume
         b       .ret_from_except
diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S
index c4c34665c221..7215cc2495df 100644
--- a/arch/powerpc/kernel/exceptions-64e.S
+++ b/arch/powerpc/kernel/exceptions-64e.S
@@ -24,6 +24,7 @@
 #include <asm/ptrace.h>
 #include <asm/ppc-opcode.h>
 #include <asm/mmu.h>
+#include <asm/hw_irq.h>
 
 /* XXX This will ultimately add space for a special exception save
  *     structure used to save things like SRR0/SRR1, SPRGs, MAS, etc...
@@ -77,59 +78,55 @@
 #define SPRN_MC_SRR1    SPRN_MCSRR1
 
 #define NORMAL_EXCEPTION_PROLOG(n, addition)                                \
-        EXCEPTION_PROLOG(n, GEN, addition##_GEN)
+        EXCEPTION_PROLOG(n, GEN, addition##_GEN(n))
 
 #define CRIT_EXCEPTION_PROLOG(n, addition)                                  \
-        EXCEPTION_PROLOG(n, CRIT, addition##_CRIT)
+        EXCEPTION_PROLOG(n, CRIT, addition##_CRIT(n))
 
 #define DBG_EXCEPTION_PROLOG(n, addition)                                   \
-        EXCEPTION_PROLOG(n, DBG, addition##_DBG)
+        EXCEPTION_PROLOG(n, DBG, addition##_DBG(n))
 
 #define MC_EXCEPTION_PROLOG(n, addition)                                    \
-        EXCEPTION_PROLOG(n, MC, addition##_MC)
+        EXCEPTION_PROLOG(n, MC, addition##_MC(n))
 
 
 /* Variants of the "addition" argument for the prolog
  */
-#define PROLOG_ADDITION_NONE_GEN
-#define PROLOG_ADDITION_NONE_CRIT
-#define PROLOG_ADDITION_NONE_DBG
-#define PROLOG_ADDITION_NONE_MC
+#define PROLOG_ADDITION_NONE_GEN(n)
+#define PROLOG_ADDITION_NONE_CRIT(n)
+#define PROLOG_ADDITION_NONE_DBG(n)
+#define PROLOG_ADDITION_NONE_MC(n)
 
-#define PROLOG_ADDITION_MASKABLE_GEN                                        \
+#define PROLOG_ADDITION_MASKABLE_GEN(n)                                     \
         lbz     r11,PACASOFTIRQEN(r13); /* are irqs soft-disabled ? */      \
         cmpwi   cr0,r11,0;              /* yes -> go out of line */         \
-        beq     masked_interrupt_book3e;
+        beq     masked_interrupt_book3e_##n
 
-#define PROLOG_ADDITION_2REGS_GEN                                           \
+#define PROLOG_ADDITION_2REGS_GEN(n)                                        \
         std     r14,PACA_EXGEN+EX_R14(r13);                                 \
         std     r15,PACA_EXGEN+EX_R15(r13)
 
-#define PROLOG_ADDITION_1REG_GEN                                            \
+#define PROLOG_ADDITION_1REG_GEN(n)                                         \
         std     r14,PACA_EXGEN+EX_R14(r13);
 
-#define PROLOG_ADDITION_2REGS_CRIT                                          \
+#define PROLOG_ADDITION_2REGS_CRIT(n)                                       \
         std     r14,PACA_EXCRIT+EX_R14(r13);                                \
         std     r15,PACA_EXCRIT+EX_R15(r13)
 
-#define PROLOG_ADDITION_2REGS_DBG                                           \
+#define PROLOG_ADDITION_2REGS_DBG(n)                                        \
         std     r14,PACA_EXDBG+EX_R14(r13);                                 \
         std     r15,PACA_EXDBG+EX_R15(r13)
 
-#define PROLOG_ADDITION_2REGS_MC                                            \
+#define PROLOG_ADDITION_2REGS_MC(n)                                         \
         std     r14,PACA_EXMC+EX_R14(r13);                                  \
         std     r15,PACA_EXMC+EX_R15(r13)
 
-#define PROLOG_ADDITION_DOORBELL_GEN                                        \
-        lbz     r11,PACASOFTIRQEN(r13); /* are irqs soft-disabled ? */      \
-        cmpwi   cr0,r11,0;              /* yes -> go out of line */         \
-        beq     masked_doorbell_book3e
-
 
 /* Core exception code for all exceptions except TLB misses.
  * XXX: Needs to make SPRN_SPRG_GEN depend on exception type
  */
 #define EXCEPTION_COMMON(n, excf, ints)                                     \
+exc_##n##_common:                                                           \
         std     r0,GPR0(r1);            /* save r0 in stackframe */         \
         std     r2,GPR2(r1);            /* save r2 in stackframe */         \
         SAVE_4GPRS(3, r1);              /* save r3 - r6 in stackframe */    \
@@ -167,20 +164,21 @@
         std     r0,RESULT(r1);          /* clear regs->result */            \
         ints;
 
-/* Variants for the "ints" argument */
+/* Variants for the "ints" argument. This one does nothing when we want
+ * to keep interrupts in their original state
+ */
 #define INTS_KEEP
-#define INTS_DISABLE_SOFT                                                   \
-        stb     r0,PACASOFTIRQEN(r13);  /* mark interrupts soft-disabled */ \
-        TRACE_DISABLE_INTS;
-#define INTS_DISABLE_HARD                                                   \
-        stb     r0,PACAHARDIRQEN(r13); /* and hard disabled */
-#define INTS_DISABLE_ALL                                                    \
-        INTS_DISABLE_SOFT                                                   \
-        INTS_DISABLE_HARD
-
-/* This is called by exceptions that used INTS_KEEP (that is did not clear
- * neither soft nor hard IRQ indicators in the PACA. This will restore MSR:EE
- * to it's previous value
+
+/* This second version is meant for exceptions that don't immediately
+ * hard-enable. We set a bit in paca->irq_happened to ensure that
+ * a subsequent call to arch_local_irq_restore() will properly
+ * hard-enable and avoid the fast-path
+ */
+#define INTS_DISABLE    SOFT_DISABLE_INTS(r3,r4)
+
+/* This is called by exceptions that used INTS_KEEP (that did not touch
+ * irq indicators in the PACA). This will restore MSR:EE to it's previous
+ * value
  *
  * XXX In the long run, we may want to open-code it in order to separate the
  *     load from the wrtee, thus limiting the latency caused by the dependency
@@ -238,7 +236,7 @@ exc_##n##_bad_stack:							    \
 #define MASKABLE_EXCEPTION(trapnum, label, hdlr, ack)                   \
         START_EXCEPTION(label);                                         \
         NORMAL_EXCEPTION_PROLOG(trapnum, PROLOG_ADDITION_MASKABLE)      \
-        EXCEPTION_COMMON(trapnum, PACA_EXGEN, INTS_DISABLE_ALL)         \
+        EXCEPTION_COMMON(trapnum, PACA_EXGEN, INTS_DISABLE)             \
         ack(r8);                                                        \
         CHECK_NAPPING();                                                \
         addi    r3,r1,STACK_FRAME_OVERHEAD;                             \
@@ -289,7 +287,7 @@ interrupt_end_book3e:
 /* Critical Input Interrupt */
         START_EXCEPTION(critical_input);
         CRIT_EXCEPTION_PROLOG(0x100, PROLOG_ADDITION_NONE)
-//      EXCEPTION_COMMON(0x100, PACA_EXCRIT, INTS_DISABLE_ALL)
+//      EXCEPTION_COMMON(0x100, PACA_EXCRIT, INTS_DISABLE)
 //      bl      special_reg_save_crit
 //      CHECK_NAPPING();
 //      addi    r3,r1,STACK_FRAME_OVERHEAD
@@ -300,7 +298,7 @@ interrupt_end_book3e:
 /* Machine Check Interrupt */
         START_EXCEPTION(machine_check);
         CRIT_EXCEPTION_PROLOG(0x200, PROLOG_ADDITION_NONE)
-//      EXCEPTION_COMMON(0x200, PACA_EXMC, INTS_DISABLE_ALL)
+//      EXCEPTION_COMMON(0x200, PACA_EXMC, INTS_DISABLE)
 //      bl      special_reg_save_mc
 //      addi    r3,r1,STACK_FRAME_OVERHEAD
 //      CHECK_NAPPING();
@@ -313,7 +311,7 @@ interrupt_end_book3e:
         NORMAL_EXCEPTION_PROLOG(0x300, PROLOG_ADDITION_2REGS)
         mfspr   r14,SPRN_DEAR
         mfspr   r15,SPRN_ESR
-        EXCEPTION_COMMON(0x300, PACA_EXGEN, INTS_DISABLE_ALL)
+        EXCEPTION_COMMON(0x300, PACA_EXGEN, INTS_DISABLE)
         b       storage_fault_common
 
 /* Instruction Storage Interrupt */
@@ -321,7 +319,7 @@ interrupt_end_book3e:
         NORMAL_EXCEPTION_PROLOG(0x400, PROLOG_ADDITION_2REGS)
         li      r15,0
         mr      r14,r10
-        EXCEPTION_COMMON(0x400, PACA_EXGEN, INTS_DISABLE_ALL)
+        EXCEPTION_COMMON(0x400, PACA_EXGEN, INTS_DISABLE)
         b       storage_fault_common
 
 /* External Input Interrupt */
@@ -339,12 +337,11 @@ interrupt_end_book3e:
         START_EXCEPTION(program);
         NORMAL_EXCEPTION_PROLOG(0x700, PROLOG_ADDITION_1REG)
         mfspr   r14,SPRN_ESR
-        EXCEPTION_COMMON(0x700, PACA_EXGEN, INTS_DISABLE_SOFT)
+        EXCEPTION_COMMON(0x700, PACA_EXGEN, INTS_DISABLE)
         std     r14,_DSISR(r1)
         addi    r3,r1,STACK_FRAME_OVERHEAD
         ld      r14,PACA_EXGEN+EX_R14(r13)
         bl      .save_nvgprs
-        INTS_RESTORE_HARD
         bl      .program_check_exception
         b       .ret_from_except
 
@@ -358,7 +355,7 @@ interrupt_end_book3e:
         beq-    1f
         bl      .load_up_fpu
         b       fast_exception_return
-1:      INTS_DISABLE_ALL
+1:      INTS_DISABLE
         bl      .save_nvgprs
         addi    r3,r1,STACK_FRAME_OVERHEAD
         bl      .kernel_fp_unavailable_exception
@@ -373,7 +370,7 @@ interrupt_end_book3e:
 /* Watchdog Timer Interrupt */
         START_EXCEPTION(watchdog);
         CRIT_EXCEPTION_PROLOG(0x9f0, PROLOG_ADDITION_NONE)
-//      EXCEPTION_COMMON(0x9f0, PACA_EXCRIT, INTS_DISABLE_ALL)
+//      EXCEPTION_COMMON(0x9f0, PACA_EXCRIT, INTS_DISABLE)
 //      bl      special_reg_save_crit
 //      CHECK_NAPPING();
 //      addi    r3,r1,STACK_FRAME_OVERHEAD
@@ -392,7 +389,7 @@ interrupt_end_book3e:
 /* Auxiliary Processor Unavailable Interrupt */
         START_EXCEPTION(ap_unavailable);
         NORMAL_EXCEPTION_PROLOG(0xf20, PROLOG_ADDITION_NONE)
-        EXCEPTION_COMMON(0xf20, PACA_EXGEN, INTS_DISABLE_ALL)
+        EXCEPTION_COMMON(0xf20, PACA_EXGEN, INTS_DISABLE)
         bl      .save_nvgprs
         addi    r3,r1,STACK_FRAME_OVERHEAD
         bl      .unknown_exception
@@ -450,7 +447,7 @@ interrupt_end_book3e:
         mfspr   r15,SPRN_SPRG_CRIT_SCRATCH
         mtspr   SPRN_SPRG_GEN_SCRATCH,r15
         mfspr   r14,SPRN_DBSR
-        EXCEPTION_COMMON(0xd00, PACA_EXCRIT, INTS_DISABLE_ALL)
+        EXCEPTION_COMMON(0xd00, PACA_EXCRIT, INTS_DISABLE)
         std     r14,_DSISR(r1)
         addi    r3,r1,STACK_FRAME_OVERHEAD
         mr      r4,r14
@@ -465,7 +462,7 @@ kernel_dbg_exc:
 
 /* Debug exception as a debug interrupt*/
         START_EXCEPTION(debug_debug);
-        DBG_EXCEPTION_PROLOG(0xd00, PROLOG_ADDITION_2REGS)
+        DBG_EXCEPTION_PROLOG(0xd08, PROLOG_ADDITION_2REGS)
 
         /*
          * If there is a single step or branch-taken exception in an
@@ -515,7 +512,7 @@ kernel_dbg_exc:
         mfspr   r15,SPRN_SPRG_DBG_SCRATCH
         mtspr   SPRN_SPRG_GEN_SCRATCH,r15
         mfspr   r14,SPRN_DBSR
-        EXCEPTION_COMMON(0xd00, PACA_EXDBG, INTS_DISABLE_ALL)
+        EXCEPTION_COMMON(0xd08, PACA_EXDBG, INTS_DISABLE)
         std     r14,_DSISR(r1)
         addi    r3,r1,STACK_FRAME_OVERHEAD
         mr      r4,r14
@@ -525,21 +522,20 @@ kernel_dbg_exc:
         bl      .DebugException
         b       .ret_from_except
 
-        MASKABLE_EXCEPTION(0x260, perfmon, .performance_monitor_exception, ACK_NONE)
-
-/* Doorbell interrupt */
-        START_EXCEPTION(doorbell)
-        NORMAL_EXCEPTION_PROLOG(0x2070, PROLOG_ADDITION_DOORBELL)
-        EXCEPTION_COMMON(0x2070, PACA_EXGEN, INTS_DISABLE_ALL)
-        CHECK_NAPPING()
+        START_EXCEPTION(perfmon);
+        NORMAL_EXCEPTION_PROLOG(0x260, PROLOG_ADDITION_NONE)
+        EXCEPTION_COMMON(0x260, PACA_EXGEN, INTS_DISABLE)
         addi    r3,r1,STACK_FRAME_OVERHEAD
-        bl      .doorbell_exception
+        bl      .performance_monitor_exception
         b       .ret_from_except_lite
 
+/* Doorbell interrupt */
+        MASKABLE_EXCEPTION(0x280, doorbell, .doorbell_exception, ACK_NONE)
+
 /* Doorbell critical Interrupt */
         START_EXCEPTION(doorbell_crit);
-        CRIT_EXCEPTION_PROLOG(0x2080, PROLOG_ADDITION_NONE)
-//      EXCEPTION_COMMON(0x2080, PACA_EXCRIT, INTS_DISABLE_ALL)
+        CRIT_EXCEPTION_PROLOG(0x2a0, PROLOG_ADDITION_NONE)
+//      EXCEPTION_COMMON(0x2a0, PACA_EXCRIT, INTS_DISABLE)
 //      bl      special_reg_save_crit
 //      CHECK_NAPPING();
 //      addi    r3,r1,STACK_FRAME_OVERHEAD
@@ -547,36 +543,114 @@ kernel_dbg_exc:
 //      b       ret_from_crit_except
         b       .
 
+/* Guest Doorbell */
         MASKABLE_EXCEPTION(0x2c0, guest_doorbell, .unknown_exception, ACK_NONE)
-        MASKABLE_EXCEPTION(0x2e0, guest_doorbell_crit, .unknown_exception, ACK_NONE)
-        MASKABLE_EXCEPTION(0x310, hypercall, .unknown_exception, ACK_NONE)
-        MASKABLE_EXCEPTION(0x320, ehpriv, .unknown_exception, ACK_NONE)
 
+/* Guest Doorbell critical Interrupt */
+        START_EXCEPTION(guest_doorbell_crit);
+        CRIT_EXCEPTION_PROLOG(0x2e0, PROLOG_ADDITION_NONE)
+//      EXCEPTION_COMMON(0x2e0, PACA_EXCRIT, INTS_DISABLE)
+//      bl      special_reg_save_crit
+//      CHECK_NAPPING();
+//      addi    r3,r1,STACK_FRAME_OVERHEAD
+//      bl      .guest_doorbell_critical_exception
+//      b       ret_from_crit_except
+        b       .
+
+/* Hypervisor call */
+        START_EXCEPTION(hypercall);
+        NORMAL_EXCEPTION_PROLOG(0x310, PROLOG_ADDITION_NONE)
+        EXCEPTION_COMMON(0x310, PACA_EXGEN, INTS_KEEP)
+        addi    r3,r1,STACK_FRAME_OVERHEAD
+        bl      .save_nvgprs
+        INTS_RESTORE_HARD
+        bl      .unknown_exception
+        b       .ret_from_except
+
+/* Embedded Hypervisor priviledged  */
+        START_EXCEPTION(ehpriv);
+        NORMAL_EXCEPTION_PROLOG(0x320, PROLOG_ADDITION_NONE)
+        EXCEPTION_COMMON(0x320, PACA_EXGEN, INTS_KEEP)
+        addi    r3,r1,STACK_FRAME_OVERHEAD
+        bl      .save_nvgprs
+        INTS_RESTORE_HARD
+        bl      .unknown_exception
+        b       .ret_from_except
 
 /*
- * An interrupt came in while soft-disabled; clear EE in SRR1,
- * clear paca->hard_enabled and return.
+ * An interrupt came in while soft-disabled; We mark paca->irq_happened
+ * accordingly and if the interrupt is level sensitive, we hard disable
  */
-masked_doorbell_book3e:
-        mtcr    r10
-        /* Resend the doorbell to fire again when ints enabled */
-        mfspr   r10,SPRN_PIR
-        PPC_MSGSND(r10)
-        b       masked_interrupt_book3e_common
 
-masked_interrupt_book3e:
+masked_interrupt_book3e_0x500:
+        /* XXX When adding support for EPR, use PACA_IRQ_EE_EDGE */
+        li      r11,PACA_IRQ_EE
+        b       masked_interrupt_book3e_full_mask
+
+masked_interrupt_book3e_0x900:
+        ACK_DEC(r11);
+        li      r11,PACA_IRQ_DEC
+        b       masked_interrupt_book3e_no_mask
+masked_interrupt_book3e_0x980:
+        ACK_FIT(r11);
+        li      r11,PACA_IRQ_DEC
+        b       masked_interrupt_book3e_no_mask
+masked_interrupt_book3e_0x280:
+masked_interrupt_book3e_0x2c0:
+        li      r11,PACA_IRQ_DBELL
+        b       masked_interrupt_book3e_no_mask
+
+masked_interrupt_book3e_no_mask:
         mtcr    r10
-masked_interrupt_book3e_common:
-        stb     r11,PACAHARDIRQEN(r13)
+        lbz     r10,PACAIRQHAPPENED(r13)
+        or      r10,r10,r11
+        stb     r10,PACAIRQHAPPENED(r13)
+        b       1f
+masked_interrupt_book3e_full_mask:
+        mtcr    r10
+        lbz     r10,PACAIRQHAPPENED(r13)
+        or      r10,r10,r11
+        stb     r10,PACAIRQHAPPENED(r13)
         mfspr   r10,SPRN_SRR1
         rldicl  r11,r10,48,1            /* clear MSR_EE */
         rotldi  r10,r11,16
         mtspr   SPRN_SRR1,r10
-        ld      r10,PACA_EXGEN+EX_R10(r13);     /* restore registers */
+1:      ld      r10,PACA_EXGEN+EX_R10(r13);
         ld      r11,PACA_EXGEN+EX_R11(r13);
         mfspr   r13,SPRN_SPRG_GEN_SCRATCH;
         rfi
         b       .
+/*
+ * Called from arch_local_irq_enable when an interrupt needs
+ * to be resent. r3 contains either 0x500,0x900,0x260 or 0x280
+ * to indicate the kind of interrupt. MSR:EE is already off.
+ * We generate a stackframe like if a real interrupt had happened.
+ *
+ * Note: While MSR:EE is off, we need to make sure that _MSR
+ * in the generated frame has EE set to 1 or the exception
+ * handler will not properly re-enable them.
+ */
+_GLOBAL(__replay_interrupt)
+        /* We are going to jump to the exception common code which
+         * will retrieve various register values from the PACA which
+         * we don't give a damn about.
+         */
+        mflr    r10
+        mfmsr   r11
+        mfcr    r4
+        mtspr   SPRN_SPRG_GEN_SCRATCH,r13;
+        std     r1,PACA_EXGEN+EX_R1(r13);
+        stw     r4,PACA_EXGEN+EX_CR(r13);
+        ori     r11,r11,MSR_EE
+        subi    r1,r1,INT_FRAME_SIZE;
+        cmpwi   cr0,r3,0x500
+        beq     exc_0x500_common
+        cmpwi   cr0,r3,0x900
+        beq     exc_0x900_common
+        cmpwi   cr0,r3,0x280
+        beq     exc_0x280_common
+        blr
+
 
 /*
  * This is called from 0x300 and 0x400 handlers after the prologs with
@@ -679,6 +753,8 @@ BAD_STACK_TRAMPOLINE(0x000)
 BAD_STACK_TRAMPOLINE(0x100)
 BAD_STACK_TRAMPOLINE(0x200)
 BAD_STACK_TRAMPOLINE(0x260)
+BAD_STACK_TRAMPOLINE(0x280)
+BAD_STACK_TRAMPOLINE(0x2a0)
 BAD_STACK_TRAMPOLINE(0x2c0)
 BAD_STACK_TRAMPOLINE(0x2e0)
 BAD_STACK_TRAMPOLINE(0x300)
@@ -696,11 +772,10 @@ BAD_STACK_TRAMPOLINE(0xa00)
 BAD_STACK_TRAMPOLINE(0xb00)
 BAD_STACK_TRAMPOLINE(0xc00)
 BAD_STACK_TRAMPOLINE(0xd00)
+BAD_STACK_TRAMPOLINE(0xd08)
 BAD_STACK_TRAMPOLINE(0xe00)
 BAD_STACK_TRAMPOLINE(0xf00)
 BAD_STACK_TRAMPOLINE(0xf20)
-BAD_STACK_TRAMPOLINE(0x2070)
-BAD_STACK_TRAMPOLINE(0x2080)
 
         .globl  bad_stack_book3e
 bad_stack_book3e:
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index 02448ea58ad3..2d0868a4e2f0 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -12,6 +12,7 @@
  *
  */
 
+#include <asm/hw_irq.h>
 #include <asm/exception-64s.h>
 #include <asm/ptrace.h>
 
@@ -356,34 +357,60 @@ do_stab_bolted_pSeries:
         KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xf40)
 
 /*
- * An interrupt came in while soft-disabled; clear EE in SRR1,
- * clear paca->hard_enabled and return.
+ * An interrupt came in while soft-disabled. We set paca->irq_happened,
+ * then, if it was a decrementer interrupt, we bump the dec to max and
+ * and return, else we hard disable and return. This is called with
+ * r10 containing the value to OR to the paca field.
  */
-masked_interrupt:
-        stb     r10,PACAHARDIRQEN(r13)
-        mtcrf   0x80,r9
-        ld      r9,PACA_EXGEN+EX_R9(r13)
-        mfspr   r10,SPRN_SRR1
-        rldicl  r10,r10,48,1            /* clear MSR_EE */
-        rotldi  r10,r10,16
-        mtspr   SPRN_SRR1,r10
-        ld      r10,PACA_EXGEN+EX_R10(r13)
-        GET_SCRATCH0(r13)
-        rfid
+#define MASKED_INTERRUPT(_H)                            \
+masked_##_H##interrupt:                                 \
+        std     r11,PACA_EXGEN+EX_R11(r13);             \
+        lbz     r11,PACAIRQHAPPENED(r13);               \
+        or      r11,r11,r10;                            \
+        stb     r11,PACAIRQHAPPENED(r13);               \
+        andi.   r10,r10,PACA_IRQ_DEC;                   \
+        beq     1f;                                     \
+        lis     r10,0x7fff;                             \
+        ori     r10,r10,0xffff;                         \
+        mtspr   SPRN_DEC,r10;                           \
+        b       2f;                                     \
+1:      mfspr   r10,SPRN_##_H##SRR1;                    \
+        rldicl  r10,r10,48,1; /* clear MSR_EE */        \
+        rotldi  r10,r10,16;                             \
+        mtspr   SPRN_##_H##SRR1,r10;                    \
+2:      mtcrf   0x80,r9;                                \
+        ld      r9,PACA_EXGEN+EX_R9(r13);               \
+        ld      r10,PACA_EXGEN+EX_R10(r13);             \
+        ld      r11,PACA_EXGEN+EX_R11(r13);             \
+        GET_SCRATCH0(r13);                              \
+        ##_H##rfid;                                     \
         b       .
+        
+        MASKED_INTERRUPT()
+        MASKED_INTERRUPT(H)
 
-masked_Hinterrupt:
-        stb     r10,PACAHARDIRQEN(r13)
-        mtcrf   0x80,r9
-        ld      r9,PACA_EXGEN+EX_R9(r13)
-        mfspr   r10,SPRN_HSRR1
-        rldicl  r10,r10,48,1            /* clear MSR_EE */
-        rotldi  r10,r10,16
-        mtspr   SPRN_HSRR1,r10
-        ld      r10,PACA_EXGEN+EX_R10(r13)
-        GET_SCRATCH0(r13)
-        hrfid
-        b       .
+/*
+ * Called from arch_local_irq_enable when an interrupt needs
+ * to be resent. r3 contains 0x500 or 0x900 to indicate which
+ * kind of interrupt. MSR:EE is already off. We generate a
+ * stackframe like if a real interrupt had happened.
+ *
+ * Note: While MSR:EE is off, we need to make sure that _MSR
+ * in the generated frame has EE set to 1 or the exception
+ * handler will not properly re-enable them.
+ */
+_GLOBAL(__replay_interrupt)
+        /* We are going to jump to the exception common code which
+         * will retrieve various register values from the PACA which
+         * we don't give a damn about, so we don't bother storing them.
+         */
+        mfmsr   r12
+        mflr    r11
+        mfcr    r9
+        ori     r12,r12,MSR_EE
+        andi.   r3,r3,0x0800
+        bne     decrementer_common
+        b       hardware_interrupt_common
 
 #ifdef CONFIG_PPC_PSERIES
 /*
@@ -793,7 +820,8 @@ vsx_unavailable_common:
         EXCEPTION_PROLOG_COMMON(0xf40, PACA_EXGEN)
 #ifdef CONFIG_VSX
 BEGIN_FTR_SECTION
-        bne     .load_up_vsx
+        beq     1f
+        b       .load_up_vsx
 1:
 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
 #endif
@@ -808,65 +836,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_VSX)
 __end_handlers:
 
 /*
- * Return from an exception with minimal checks.
- * The caller is assumed to have done EXCEPTION_PROLOG_COMMON.
- * If interrupts have been enabled, or anything has been
- * done that might have changed the scheduling status of
- * any task or sent any task a signal, you should use
- * ret_from_except or ret_from_except_lite instead of this.
- */
-fast_exc_return_irq:                    /* restores irq state too */
-        ld      r3,SOFTE(r1)
-        TRACE_AND_RESTORE_IRQ(r3);
-        ld      r12,_MSR(r1)
-        rldicl  r4,r12,49,63            /* get MSR_EE to LSB */
-        stb     r4,PACAHARDIRQEN(r13)   /* restore paca->hard_enabled */
-        b       1f
-
-        .globl  fast_exception_return
-fast_exception_return:
-        ld      r12,_MSR(r1)
-1:      ld      r11,_NIP(r1)
-        andi.   r3,r12,MSR_RI           /* check if RI is set */
-        beq-    unrecov_fer
-
-#ifdef CONFIG_VIRT_CPU_ACCOUNTING
-        andi.   r3,r12,MSR_PR
-        beq     2f
-        ACCOUNT_CPU_USER_EXIT(r3, r4)
-2:
-#endif
-
-        ld      r3,_CCR(r1)
-        ld      r4,_LINK(r1)
-        ld      r5,_CTR(r1)
-        ld      r6,_XER(r1)
-        mtcr    r3
-        mtlr    r4
-        mtctr   r5
-        mtxer   r6
-        REST_GPR(0, r1)
-        REST_8GPRS(2, r1)
-
-        ld      r10,PACAKMSR(r13)
-        clrrdi  r10,r10,2               /* clear RI */
-        mtmsrd  r10,1
-
-        mtspr   SPRN_SRR1,r12
-        mtspr   SPRN_SRR0,r11
-        REST_4GPRS(10, r1)
-        ld      r1,GPR1(r1)
-        rfid
-        b       .       /* prevent speculative execution */
-
-unrecov_fer:
-        bl      .save_nvgprs
-1:      addi    r3,r1,STACK_FRAME_OVERHEAD
-        bl      .unrecoverable_exception
-        b       1b
-
-
-/*
  * Hash table stuff
  */
         .align  7
@@ -905,19 +874,16 @@ END_MMU_FTR_SECTION_IFCLR(MMU_FTR_SLB)
          * r4 contains the required access permissions
          * r5 contains the trap number
          *
-         * at return r3 = 0 for success
+         * at return r3 = 0 for success, 1 for page fault, negative for error
          */
         bl      .hash_page              /* build HPTE if possible */
         cmpdi   r3,0                    /* see if hash_page succeeded */
 
-        /*
-         * Here we have interrupts hard-disabled, so it is sufficient
-         * to restore paca->{soft,hard}_enable and get out.
-         */
+        /* Success */
         beq     fast_exc_return_irq     /* Return from exception on success */
 
-        /* For a hash failure, we don't bother re-enabling interrupts */
-        ble-    13f
+        /* Error */
+        blt-    13f
 
 /* Here we have a page fault that hash_page can't handle. */
 handle_page_fault:
diff --git a/arch/powerpc/kernel/head_64.S b/arch/powerpc/kernel/head_64.S
index 40759fbfb171..58bddee8e1e8 100644
--- a/arch/powerpc/kernel/head_64.S
+++ b/arch/powerpc/kernel/head_64.S
@@ -38,6 +38,7 @@
 #include <asm/irqflags.h>
 #include <asm/kvm_book3s_asm.h>
 #include <asm/ptrace.h>
+#include <asm/hw_irq.h>
 
 /* The physical memory is laid out such that the secondary processor
  * spin code sits at 0x0000...0x00ff. On server, the vectors follow
@@ -550,7 +551,8 @@ _GLOBAL(pmac_secondary_start)
          */
         li      r0,0
         stb     r0,PACASOFTIRQEN(r13)
-        stb     r0,PACAHARDIRQEN(r13)
+        li      r0,PACA_IRQ_HARD_DIS
+        stb     r0,PACAIRQHAPPENED(r13)
 
         /* Create a temp kernel stack for use before relocation is on.  */
         ld      r1,PACAEMERGSP(r13)
@@ -601,9 +603,12 @@ __secondary_start:
         li      r7,0
         mtlr    r7
 
-        /* Mark interrupts both hard and soft disabled */
-        stb     r7,PACAHARDIRQEN(r13)
+        /* Mark interrupts soft and hard disabled (they might be enabled
+         * in the PACA when doing hotplug)
+         */
         stb     r7,PACASOFTIRQEN(r13)
+        li      r0,PACA_IRQ_HARD_DIS
+        stb     r0,PACAIRQHAPPENED(r13)
 
         /* enable MMU and jump to start_secondary */
         LOAD_REG_ADDR(r3, .start_secondary_prolog)
@@ -750,13 +755,18 @@ _INIT_GLOBAL(start_here_common)
         /* Load the TOC (virtual address) */
         ld      r2,PACATOC(r13)
 
+        /* Do more system initializations in virtual mode */
         bl      .setup_system
 
-        /* Load up the kernel context */
-5:      li      r5,0
-        stb     r5,PACASOFTIRQEN(r13)   /* Soft Disabled */
-        stb     r5,PACAHARDIRQEN(r13)   /* Hard Disabled on others */
+        /* Mark interrupts soft and hard disabled (they might be enabled
+         * in the PACA when doing hotplug)
+         */
+        li      r0,0
+        stb     r0,PACASOFTIRQEN(r13)
+        li      r0,PACA_IRQ_HARD_DIS
+        stb     r0,PACAIRQHAPPENED(r13)
 
+        /* Generic kernel entry */
         bl      .start_kernel
 
         /* Not reached */
diff --git a/arch/powerpc/kernel/idle.c b/arch/powerpc/kernel/idle.c
index 0a48bf5db6c8..8f7a2b62863d 100644
--- a/arch/powerpc/kernel/idle.c
+++ b/arch/powerpc/kernel/idle.c
@@ -84,7 +84,11 @@ void cpu_idle(void)
 
                                 start_critical_timings();
 
-                                local_irq_enable();
+                                /* Some power_save functions return with
+                                 * interrupts enabled, some don't.
+                                 */
+                                if (irqs_disabled())
+                                        local_irq_enable();
                                 set_thread_flag(TIF_POLLING_NRFLAG);
 
                         } else {
diff --git a/arch/powerpc/kernel/idle_book3e.S b/arch/powerpc/kernel/idle_book3e.S
index 16c002d6bdf1..ff007b59448d 100644
--- a/arch/powerpc/kernel/idle_book3e.S
+++ b/arch/powerpc/kernel/idle_book3e.S
@@ -29,43 +29,30 @@ _GLOBAL(book3e_idle)
         wrteei  0
 
         /* Now check if an interrupt came in while we were soft disabled
-         * since we may otherwise lose it (doorbells etc...). We know
-         * that since PACAHARDIRQEN will have been cleared in that case.
+         * since we may otherwise lose it (doorbells etc...).
          */
-        lbz     r3,PACAHARDIRQEN(r13)
+        lbz     r3,PACAIRQHAPPENED(r13)
         cmpwi   cr0,r3,0
-        beqlr
+        bnelr
 
-        /* Now we are going to mark ourselves as soft and hard enables in
+        /* Now we are going to mark ourselves as soft and hard enabled in
          * order to be able to take interrupts while asleep. We inform lockdep
          * of that. We don't actually turn interrupts on just yet tho.
          */
 #ifdef CONFIG_TRACE_IRQFLAGS
         stdu    r1,-128(r1)
         bl      .trace_hardirqs_on
+        addi    r1,r1,128
 #endif
         li      r0,1
         stb     r0,PACASOFTIRQEN(r13)
-        stb     r0,PACAHARDIRQEN(r13)
         
         /* Interrupts will make use return to LR, so get something we want
          * in there
          */
         bl      1f
 
-        /* Hard disable interrupts again */
-        wrteei  0
-
-        /* Mark them off again in the PACA as well */
-        li      r0,0
-        stb     r0,PACASOFTIRQEN(r13)
-        stb     r0,PACAHARDIRQEN(r13)
-
-        /* Tell lockdep about it */
-#ifdef CONFIG_TRACE_IRQFLAGS
-        bl      .trace_hardirqs_off
-        addi    r1,r1,128
-#endif
+        /* And return (interrupts are on) */
         ld      r0,16(r1)
         mtlr    r0
         blr
diff --git a/arch/powerpc/kernel/idle_power4.S b/arch/powerpc/kernel/idle_power4.S
index ba3195478600..d8cdba4c28b2 100644
--- a/arch/powerpc/kernel/idle_power4.S
+++ b/arch/powerpc/kernel/idle_power4.S
@@ -14,6 +14,7 @@
 #include <asm/thread_info.h>
 #include <asm/ppc_asm.h>
 #include <asm/asm-offsets.h>
+#include <asm/irqflags.h>
 
 #undef DEBUG
 
@@ -29,14 +30,31 @@ END_FTR_SECTION_IFCLR(CPU_FTR_CAN_NAP)
         cmpwi   0,r4,0
         beqlr
 
-        /* Go to NAP now */
+        /* Hard disable interrupts */
         mfmsr   r7
         rldicl  r0,r7,48,1
         rotldi  r0,r0,16
-        mtmsrd  r0,1                    /* hard-disable interrupts */
+        mtmsrd  r0,1
+
+        /* Check if something happened while soft-disabled */
+        lbz     r0,PACAIRQHAPPENED(r13)
+        cmpwi   cr0,r0,0
+        bnelr
+
+        /* Soft-enable interrupts */
+#ifdef CONFIG_TRACE_IRQFLAGS
+        mflr    r0
+        std     r0,16(r1)
+        stdu    r1,-128(r1)
+        bl      .trace_hardirqs_on
+        addi    r1,r1,128
+        ld      r0,16(r1)
+        mtlr    r0
+#endif /* CONFIG_TRACE_IRQFLAGS */
+
+        TRACE_ENABLE_INTS
         li      r0,1
         stb     r0,PACASOFTIRQEN(r13)   /* we'll hard-enable shortly */
-        stb     r0,PACAHARDIRQEN(r13)
 BEGIN_FTR_SECTION
         DSSALL
         sync
diff --git a/arch/powerpc/kernel/idle_power7.S b/arch/powerpc/kernel/idle_power7.S
index fcdff198da4b..0cdc9a392839 100644
--- a/arch/powerpc/kernel/idle_power7.S
+++ b/arch/powerpc/kernel/idle_power7.S
@@ -1,5 +1,5 @@
 /*
- *  This file contains the power_save function for 970-family CPUs.
+ *  This file contains the power_save function for Power7 CPUs.
  *
  *  This program is free software; you can redistribute it and/or
  *  modify it under the terms of the GNU General Public License
@@ -15,6 +15,7 @@
 #include <asm/ppc_asm.h>
 #include <asm/asm-offsets.h>
 #include <asm/ppc-opcode.h>
+#include <asm/hw_irq.h>
 
 #undef DEBUG
 
@@ -51,9 +52,25 @@ _GLOBAL(power7_idle)
         rldicl  r9,r9,48,1
         rotldi  r9,r9,16
         mtmsrd  r9,1                    /* hard-disable interrupts */
+
+        /* Check if something happened while soft-disabled */
+        lbz     r0,PACAIRQHAPPENED(r13)
+        cmpwi   cr0,r0,0
+        beq     1f
+        addi    r1,r1,INT_FRAME_SIZE
+        ld      r0,16(r1)
+        mtlr    r0
+        blr
+
+1:      /* We mark irqs hard disabled as this is the state we'll
+         * be in when returning and we need to tell arch_local_irq_restore()
+         * about it
+         */
+        li      r0,PACA_IRQ_HARD_DIS
+        stb     r0,PACAIRQHAPPENED(r13)
+
+        /* We haven't lost state ... yet */
         li      r0,0
-        stb     r0,PACASOFTIRQEN(r13)   /* we'll hard-enable shortly */
-        stb     r0,PACAHARDIRQEN(r13)
         stb     r0,PACA_NAPSTATELOST(r13)
 
         /* Continue saving state */
diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c
index 9b6e80668cfb..eb804e15b29b 100644
--- a/arch/powerpc/kernel/irq.c
+++ b/arch/powerpc/kernel/irq.c
@@ -95,14 +95,14 @@ extern int tau_interrupts(int);
 
 int distribute_irqs = 1;
 
-static inline notrace unsigned long get_hard_enabled(void)
+static inline notrace unsigned long get_irq_happened(void)
 {
-        unsigned long enabled;
+        unsigned long happened;
 
         __asm__ __volatile__("lbz %0,%1(13)"
-        : "=r" (enabled) : "i" (offsetof(struct paca_struct, hard_enabled)));
+        : "=r" (happened) : "i" (offsetof(struct paca_struct, irq_happened)));
 
-        return enabled;
+        return happened;
 }
 
 static inline notrace void set_soft_enabled(unsigned long enable)
@@ -111,88 +111,167 @@ static inline notrace void set_soft_enabled(unsigned long enable)
         : : "r" (enable), "i" (offsetof(struct paca_struct, soft_enabled)));
 }
 
-static inline notrace void decrementer_check_overflow(void)
+static inline notrace int decrementer_check_overflow(void)
 {
-        u64 now = get_tb_or_rtc();
-        u64 *next_tb;
-
-        preempt_disable();
-        next_tb = &__get_cpu_var(decrementers_next_tb);
-
+        u64 now = get_tb_or_rtc();
+        u64 *next_tb = &__get_cpu_var(decrementers_next_tb);
+ 
         if (now >= *next_tb)
                 set_dec(1);
-        preempt_enable();
+        return now >= *next_tb;
 }
 
-notrace void arch_local_irq_restore(unsigned long en)
+/* This is called whenever we are re-enabling interrupts
+ * and returns either 0 (nothing to do) or 500/900 if there's
+ * either an EE or a DEC to generate.
+ *
+ * This is called in two contexts: From arch_local_irq_restore()
+ * before soft-enabling interrupts, and from the exception exit
+ * path when returning from an interrupt from a soft-disabled to
+ * a soft enabled context. In both case we have interrupts hard
+ * disabled.
+ *
+ * We take care of only clearing the bits we handled in the
+ * PACA irq_happened field since we can only re-emit one at a
+ * time and we don't want to "lose" one.
+ */
+notrace unsigned int __check_irq_replay(void)
 {
         /*
-         * get_paca()->soft_enabled = en;
-         * Is it ever valid to use local_irq_restore(0) when soft_enabled is 1?
-         * That was allowed before, and in such a case we do need to take care
-         * that gcc will set soft_enabled directly via r13, not choose to use
-         * an intermediate register, lest we're preempted to a different cpu.
+         * We use local_paca rather than get_paca() to avoid all
+         * the debug_smp_processor_id() business in this low level
+         * function
          */
-        set_soft_enabled(en);
-        if (!en)
-                return;
+        unsigned char happened = local_paca->irq_happened;
 
-#ifdef CONFIG_PPC_STD_MMU_64
-        if (firmware_has_feature(FW_FEATURE_ISERIES)) {
-                /*
-                 * Do we need to disable preemption here?  Not really: in the
-                 * unlikely event that we're preempted to a different cpu in
-                 * between getting r13, loading its lppaca_ptr, and loading
-                 * its any_int, we might call iseries_handle_interrupts without
-                 * an interrupt pending on the new cpu, but that's no disaster,
-                 * is it?  And the business of preempting us off the old cpu
-                 * would itself involve a local_irq_restore which handles the
-                 * interrupt to that cpu.
-                 *
-                 * But use "local_paca->lppaca_ptr" instead of "get_lppaca()"
-                 * to avoid any preemption checking added into get_paca().
-                 */
-                if (local_paca->lppaca_ptr->int_dword.any_int)
-                        iseries_handle_interrupts();
+        /* Clear bit 0 which we wouldn't clear otherwise */
+        local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS;
+
+        /*
+         * Force the delivery of pending soft-disabled interrupts on PS3.
+         * Any HV call will have this side effect.
+         */
+        if (firmware_has_feature(FW_FEATURE_PS3_LV1)) {
+                u64 tmp, tmp2;
+                lv1_get_version_info(&tmp, &tmp2);
         }
-#endif /* CONFIG_PPC_STD_MMU_64 */
 
         /*
-         * if (get_paca()->hard_enabled) return;
-         * But again we need to take care that gcc gets hard_enabled directly
-         * via r13, not choose to use an intermediate register, lest we're
-         * preempted to a different cpu in between the two instructions.
+         * We may have missed a decrementer interrupt. We check the
+         * decrementer itself rather than the paca irq_happened field
+         * in case we also had a rollover while hard disabled
+         */
+        local_paca->irq_happened &= ~PACA_IRQ_DEC;
+        if (decrementer_check_overflow())
+                return 0x900;
+
+        /* Finally check if an external interrupt happened */
+        local_paca->irq_happened &= ~PACA_IRQ_EE;
+        if (happened & PACA_IRQ_EE)
+                return 0x500;
+
+#ifdef CONFIG_PPC_BOOK3E
+        /* Finally check if an EPR external interrupt happened
+         * this bit is typically set if we need to handle another
+         * "edge" interrupt from within the MPIC "EPR" handler
+         */
+        local_paca->irq_happened &= ~PACA_IRQ_EE_EDGE;
+        if (happened & PACA_IRQ_EE_EDGE)
+                return 0x500;
+
+        local_paca->irq_happened &= ~PACA_IRQ_DBELL;
+        if (happened & PACA_IRQ_DBELL)
+                return 0x280;
+#endif /* CONFIG_PPC_BOOK3E */
+
+        /* There should be nothing left ! */
+        BUG_ON(local_paca->irq_happened != 0);
+
+        return 0;
+}
+
+notrace void arch_local_irq_restore(unsigned long en)
+{
+        unsigned char irq_happened;
+        unsigned int replay;
+
+        /* Write the new soft-enabled value */
+        set_soft_enabled(en);
+        if (!en)
+                return;
+        /*
+         * From this point onward, we can take interrupts, preempt,
+         * etc... unless we got hard-disabled. We check if an event
+         * happened. If none happened, we know we can just return.
+         *
+         * We may have preempted before the check below, in which case
+         * we are checking the "new" CPU instead of the old one. This
+         * is only a problem if an event happened on the "old" CPU.
+         *
+         * External interrupt events on non-iseries will have caused
+         * interrupts to be hard-disabled, so there is no problem, we
+         * cannot have preempted.
+         *
+         * That leaves us with EEs on iSeries or decrementer interrupts,
+         * which I decided to safely ignore. The preemption would have
+         * itself been the result of an interrupt, upon which return we
+         * will have checked for pending events on the old CPU.
          */
-        if (get_hard_enabled())
+        irq_happened = get_irq_happened();
+        if (!irq_happened)
                 return;
 
         /*
-         * Need to hard-enable interrupts here.  Since currently disabled,
-         * no need to take further asm precautions against preemption; but
-         * use local_paca instead of get_paca() to avoid preemption checking.
+         * We need to hard disable to get a trusted value from
+         * __check_irq_replay(). We also need to soft-disable
+         * again to avoid warnings in there due to the use of
+         * per-cpu variables.
+         *
+         * We know that if the value in irq_happened is exactly 0x01
+         * then we are already hard disabled (there are other less
+         * common cases that we'll ignore for now), so we skip the
+         * (expensive) mtmsrd.
          */
-        local_paca->hard_enabled = en;
+        if (unlikely(irq_happened != PACA_IRQ_HARD_DIS))
+                __hard_irq_disable();
+        set_soft_enabled(0);
 
         /*
-         * Trigger the decrementer if we have a pending event. Some processors
-         * only trigger on edge transitions of the sign bit. We might also
-         * have disabled interrupts long enough that the decrementer wrapped
-         * to positive.
+         * Check if anything needs to be re-emitted. We haven't
+         * soft-enabled yet to avoid warnings in decrementer_check_overflow
+         * accessing per-cpu variables
          */
-        decrementer_check_overflow();
+        replay = __check_irq_replay();
+
+        /* We can soft-enable now */
+        set_soft_enabled(1);
 
         /*
-         * Force the delivery of pending soft-disabled interrupts on PS3.
-         * Any HV call will have this side effect.
+         * And replay if we have to. This will return with interrupts
+         * hard-enabled.
          */
-        if (firmware_has_feature(FW_FEATURE_PS3_LV1)) {
-                u64 tmp, tmp2;
-                lv1_get_version_info(&tmp, &tmp2);
+        if (replay) {
+                __replay_interrupt(replay);
+                return;
         }
 
+        /* Finally, let's ensure we are hard enabled */
         __hard_irq_enable();
 }
 EXPORT_SYMBOL(arch_local_irq_restore);
+
+/*
+ * This is specifically called by assembly code to re-enable interrupts
+ * if they are currently disabled. This is typically called before
+ * schedule() or do_signal() when returning to userspace. We do it
+ * in C to avoid the burden of dealing with lockdep etc...
+ */
+void restore_interrupts(void)
+{
+        if (irqs_disabled())
+                local_irq_enable();
+}
+
 #endif /* CONFIG_PPC64 */
 
 int arch_show_interrupts(struct seq_file *p, int prec)
@@ -360,8 +439,17 @@ void do_IRQ(struct pt_regs *regs)
 
         check_stack_overflow();
 
+        /*
+         * Query the platform PIC for the interrupt & ack it.
+         *
+         * This will typically lower the interrupt line to the CPU
+         */
         irq = ppc_md.get_irq();
 
+        /* We can hard enable interrupts now */
+        may_hard_irq_enable();
+
+        /* And finally process it */
         if (irq != NO_IRQ && irq != NO_IRQ_IGNORE)
                 handle_one_irq(irq);
         else if (irq != NO_IRQ_IGNORE)
diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index bf80a1d5f8fe..e40707032ac3 100644
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -647,6 +647,9 @@ void show_regs(struct pt_regs * regs)
         printk("MSR: "REG" ", regs->msr);
         printbits(regs->msr, msr_bits);
         printk("  CR: %08lx  XER: %08lx\n", regs->ccr, regs->xer);
+#ifdef CONFIG_PPC64
+        printk("SOFTE: %ld\n", regs->softe);
+#endif
         trap = TRAP(regs);
         if ((regs->trap != 0xc00) && cpu_has_feature(CPU_FTR_CFAR))
                 printk("CFAR: "REG"\n", regs->orig_gpr3);
diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c
index 567dd7c3ac2a..f81c81b92f0e 100644
--- a/arch/powerpc/kernel/time.c
+++ b/arch/powerpc/kernel/time.c
@@ -259,7 +259,6 @@ void accumulate_stolen_time(void)
         u64 sst, ust;
 
         u8 save_soft_enabled = local_paca->soft_enabled;
-        u8 save_hard_enabled = local_paca->hard_enabled;
 
         /* We are called early in the exception entry, before
          * soft/hard_enabled are sync'ed to the expected state
@@ -268,7 +267,6 @@ void accumulate_stolen_time(void)
          * complain
          */
         local_paca->soft_enabled = 0;
-        local_paca->hard_enabled = 0;
 
         sst = scan_dispatch_log(local_paca->starttime_user);
         ust = scan_dispatch_log(local_paca->starttime);
@@ -277,7 +275,6 @@ void accumulate_stolen_time(void)
         local_paca->stolen_time += ust + sst;
 
         local_paca->soft_enabled = save_soft_enabled;
-        local_paca->hard_enabled = save_hard_enabled;
 }
 
 static inline u64 calculate_stolen_time(u64 stop_tb)
@@ -580,6 +577,11 @@ void timer_interrupt(struct pt_regs * regs)
         if (!cpu_online(smp_processor_id()))
                 return;
 
+        /* Conditionally hard-enable interrupts now that the DEC has been
+         * bumped to its maximum value
+         */
+        may_hard_irq_enable();
+
         trace_timer_interrupt_entry(regs);
 
         __get_cpu_var(irq_stat).timer_irqs++;
diff --git a/arch/powerpc/platforms/pseries/processor_idle.c b/arch/powerpc/platforms/pseries/processor_idle.c
index 085fd3f45ad2..a12e95af6933 100644
--- a/arch/powerpc/platforms/pseries/processor_idle.c
+++ b/arch/powerpc/platforms/pseries/processor_idle.c
@@ -96,6 +96,20 @@ out:
         return index;
 }
 
+static void check_and_cede_processor(void)
+{
+        /*
+         * Interrupts are soft-disabled at this point,
+         * but not hard disabled. So an interrupt might have
+         * occurred before entering NAP, and would be potentially
+         * lost (edge events, decrementer events, etc...) unless
+         * we first hard disable then check.
+         */
+        hard_irq_disable();
+        if (get_paca()->irq_happened == 0)
+                cede_processor();
+}
+
 static int dedicated_cede_loop(struct cpuidle_device *dev,
                                 struct cpuidle_driver *drv,
                                 int index)
@@ -108,7 +122,7 @@ static int dedicated_cede_loop(struct cpuidle_device *dev,
 
         ppc64_runlatch_off();
         HMT_medium();
-        cede_processor();
+        check_and_cede_processor();
 
         get_lppaca()->donate_dedicated_cpu = 0;
         dev->last_residency =
@@ -132,7 +146,7 @@ static int shared_cede_loop(struct cpuidle_device *dev,
          * processor. When returning here, external interrupts
          * are enabled.
          */
-        cede_processor();
+        check_and_cede_processor();
 
         dev->last_residency =
                 (int)idle_loop_epilog(in_purr, kt_before);
diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c
index 63846ebd3276..974a47b3c9b8 100644
--- a/arch/powerpc/xmon/xmon.c
+++ b/arch/powerpc/xmon/xmon.c
@@ -1437,8 +1437,8 @@ static void excprint(struct pt_regs *fp)
 
         printf("  current = 0x%lx\n", current);
 #ifdef CONFIG_PPC64
-        printf("  paca    = 0x%lx\t softe: %d\t harde: %d\n",
-               local_paca, local_paca->soft_enabled, local_paca->hard_enabled);
+        printf("  paca    = 0x%lx\t softe: %d\t irq_happened: 0x%02x\n",
+               local_paca, local_paca->soft_enabled, local_paca->irq_happened);
 #endif
         if (current) {
                 printf("    pid   = %ld, comm = %s\n",