8 files changed, 135 insertions, 61 deletions

diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index d06f378597bb..e96521530d21 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -118,7 +118,8 @@ int main(void)
         DEFINE(PACASTABRR, offsetof(struct paca_struct, stab_rr));
         DEFINE(PACAR1, offsetof(struct paca_struct, saved_r1));
         DEFINE(PACATOC, offsetof(struct paca_struct, kernel_toc));
-        DEFINE(PACAPROCENABLED, offsetof(struct paca_struct, proc_enabled));
+        DEFINE(PACASOFTIRQEN, offsetof(struct paca_struct, soft_enabled));
+        DEFINE(PACAHARDIRQEN, offsetof(struct paca_struct, hard_enabled));
         DEFINE(PACASLBCACHE, offsetof(struct paca_struct, slb_cache));
         DEFINE(PACASLBCACHEPTR, offsetof(struct paca_struct, slb_cache_ptr));
         DEFINE(PACACONTEXTID, offsetof(struct paca_struct, context.id));
diff --git a/arch/powerpc/kernel/crash.c b/arch/powerpc/kernel/crash.c
index 1af41f7616dc..89b03c8da9d2 100644
--- a/arch/powerpc/kernel/crash.c
+++ b/arch/powerpc/kernel/crash.c
@@ -111,7 +111,7 @@ void crash_ipi_callback(struct pt_regs *regs)
         if (!cpu_online(cpu))
                 return;
 
-        local_irq_disable();
+        hard_irq_disable();
         if (!cpu_isset(cpu, cpus_in_crash))
                 crash_save_this_cpu(regs, cpu);
         cpu_set(cpu, cpus_in_crash);
@@ -289,7 +289,7 @@ void default_machine_crash_shutdown(struct pt_regs *regs)
          * an SMP system.
          * The kernel is broken so disable interrupts.
          */
-        local_irq_disable();
+        hard_irq_disable();
 
         for_each_irq(irq) {
                 struct irq_desc *desc = irq_desc + irq;
diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S
index 748e74fcf541..efda48741b29 100644
--- a/arch/powerpc/kernel/entry_64.S
+++ b/arch/powerpc/kernel/entry_64.S
@@ -87,6 +87,10 @@ system_call_common:
         addi    r9,r1,STACK_FRAME_OVERHEAD
         ld      r11,exception_marker@toc(r2)
         std     r11,-16(r9)             /* "regshere" marker */
+        li      r10,1
+        stb     r10,PACASOFTIRQEN(r13)
+        stb     r10,PACAHARDIRQEN(r13)
+        std     r10,SOFTE(r1)
 #ifdef CONFIG_PPC_ISERIES
 BEGIN_FW_FTR_SECTION
         /* Hack for handling interrupts when soft-enabling on iSeries */
@@ -94,8 +98,6 @@ BEGIN_FW_FTR_SECTION
         andi.   r10,r12,MSR_PR          /* from kernel */
         crand   4*cr0+eq,4*cr1+eq,4*cr0+eq
         beq     hardware_interrupt_entry
-        lbz     r10,PACAPROCENABLED(r13)
-        std     r10,SOFTE(r1)
 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
 #endif
         mfmsr   r11
@@ -460,9 +462,9 @@ _GLOBAL(ret_from_except_lite)
 #endif
 
 restore:
+        ld      r5,SOFTE(r1)
 #ifdef CONFIG_PPC_ISERIES
 BEGIN_FW_FTR_SECTION
-        ld      r5,SOFTE(r1)
         cmpdi   0,r5,0
         beq     4f
         /* Check for pending interrupts (iSeries) */
@@ -472,16 +474,16 @@ BEGIN_FW_FTR_SECTION
         beq+    4f                      /* skip do_IRQ if no interrupts */
 
         li      r3,0
-        stb     r3,PACAPROCENABLED(r13) /* ensure we are soft-disabled */
+        stb     r3,PACASOFTIRQEN(r13)   /* ensure we are soft-disabled */
         ori     r10,r10,MSR_EE
         mtmsrd  r10                     /* hard-enable again */
         addi    r3,r1,STACK_FRAME_OVERHEAD
         bl      .do_IRQ
         b       .ret_from_except_lite           /* loop back and handle more */
-
-4:      stb     r5,PACAPROCENABLED(r13)
+4:
 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
 #endif
+        stb     r5,PACASOFTIRQEN(r13)
 
         ld      r3,_MSR(r1)
         andi.   r0,r3,MSR_RI
@@ -538,25 +540,15 @@ do_work:
         /* Check that preempt_count() == 0 and interrupts are enabled */
         lwz     r8,TI_PREEMPT(r9)
         cmpwi   cr1,r8,0
-#ifdef CONFIG_PPC_ISERIES
-BEGIN_FW_FTR_SECTION
         ld      r0,SOFTE(r1)
         cmpdi   r0,0
-END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
-#endif
-BEGIN_FW_FTR_SECTION
-        andi.   r0,r3,MSR_EE
-END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES)
         crandc  eq,cr1*4+eq,eq
         bne     restore
         /* here we are preempting the current task */
 1:
-#ifdef CONFIG_PPC_ISERIES
-BEGIN_FW_FTR_SECTION
         li      r0,1
-        stb     r0,PACAPROCENABLED(r13)
-END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
-#endif
+        stb     r0,PACASOFTIRQEN(r13)
+        stb     r0,PACAHARDIRQEN(r13)
         ori     r10,r10,MSR_EE
         mtmsrd  r10,1           /* reenable interrupts */
         bl      .preempt_schedule
@@ -639,8 +631,7 @@ _GLOBAL(enter_rtas)
         /* There is no way it is acceptable to get here with interrupts enabled,
          * check it with the asm equivalent of WARN_ON
          */
-        mfmsr   r6
-        andi.   r0,r6,MSR_EE
+        lbz     r0,PACASOFTIRQEN(r13)
 1:      tdnei   r0,0
 .section __bug_table,"a"
         .llong  1b,__LINE__ + 0x1000000, 1f, 2f
@@ -649,7 +640,13 @@ _GLOBAL(enter_rtas)
 1:      .asciz  __FILE__
 2:      .asciz "enter_rtas"
 .previous
-        
+
+        /* Hard-disable interrupts */
+        mfmsr   r6
+        rldicl  r7,r6,48,1
+        rotldi  r7,r7,16
+        mtmsrd  r7,1
+
         /* Unfortunately, the stack pointer and the MSR are also clobbered,
          * so they are saved in the PACA which allows us to restore
          * our original state after RTAS returns.
diff --git a/arch/powerpc/kernel/head_64.S b/arch/powerpc/kernel/head_64.S
index 645c7f10fb28..c93d9f35a121 100644
--- a/arch/powerpc/kernel/head_64.S
+++ b/arch/powerpc/kernel/head_64.S
@@ -35,9 +35,7 @@
 #include <asm/thread_info.h>
 #include <asm/firmware.h>
 
-#ifdef CONFIG_PPC_ISERIES
 #define DO_SOFT_DISABLE
-#endif
 
 /*
  * We layout physical memory as follows:
@@ -308,7 +306,9 @@ exception_marker:
         std     r9,_LINK(r1);                                              \
         mfctr   r10;                    /* save CTR in stackframe       */ \
         std     r10,_CTR(r1);                                              \
+        lbz     r10,PACASOFTIRQEN(r13);                            \
         mfspr   r11,SPRN_XER;           /* save XER in stackframe       */ \
+        std     r10,SOFTE(r1);                                             \
         std     r11,_XER(r1);                                              \
         li      r9,(n)+1;                                                  \
         std     r9,_TRAP(r1);           /* set trap number              */ \
@@ -343,6 +343,34 @@ label##_pSeries:					\
         EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common)
 
 
+#define MASKABLE_EXCEPTION_PSERIES(n, label)                            \
+        . = n;                                                          \
+        .globl label##_pSeries;                                         \
+label##_pSeries:                                                        \
+        HMT_MEDIUM;                                                     \
+        mtspr   SPRN_SPRG1,r13;         /* save r13 */                  \
+        mfspr   r13,SPRN_SPRG3;         /* get paca address into r13 */ \
+        std     r9,PACA_EXGEN+EX_R9(r13);       /* save r9, r10 */      \
+        std     r10,PACA_EXGEN+EX_R10(r13);                             \
+        lbz     r10,PACASOFTIRQEN(r13);                                 \
+        mfcr    r9;                                                     \
+        cmpwi   r10,0;                                                  \
+        beq     masked_interrupt;                                       \
+        mfspr   r10,SPRN_SPRG1;                                         \
+        std     r10,PACA_EXGEN+EX_R13(r13);                             \
+        std     r11,PACA_EXGEN+EX_R11(r13);                             \
+        std     r12,PACA_EXGEN+EX_R12(r13);                             \
+        clrrdi  r12,r13,32;             /* get high part of &label */   \
+        mfmsr   r10;                                                    \
+        mfspr   r11,SPRN_SRR0;          /* save SRR0 */                 \
+        LOAD_HANDLER(r12,label##_common)                                \
+        ori     r10,r10,MSR_IR|MSR_DR|MSR_RI;                           \
+        mtspr   SPRN_SRR0,r12;                                          \
+        mfspr   r12,SPRN_SRR1;          /* and SRR1 */                  \
+        mtspr   SPRN_SRR1,r10;                                          \
+        rfid;                                                           \
+        b       .       /* prevent speculative execution */
+
 #define STD_EXCEPTION_ISERIES(n, label, area)           \
         .globl label##_iSeries;                         \
 label##_iSeries:                                        \
@@ -358,40 +386,32 @@ label##_iSeries:							\
         HMT_MEDIUM;                                                     \
         mtspr   SPRN_SPRG1,r13;         /* save r13 */                  \
         EXCEPTION_PROLOG_ISERIES_1(PACA_EXGEN);                         \
-        lbz     r10,PACAPROCENABLED(r13);                               \
+        lbz     r10,PACASOFTIRQEN(r13);                                 \
         cmpwi   0,r10,0;                                                \
         beq-    label##_iSeries_masked;                                 \
         EXCEPTION_PROLOG_ISERIES_2;                                     \
         b       label##_common;                                         \
 
-#ifdef DO_SOFT_DISABLE
+#ifdef CONFIG_PPC_ISERIES
 #define DISABLE_INTS                            \
-BEGIN_FW_FTR_SECTION;                           \
-        lbz     r10,PACAPROCENABLED(r13);       \
         li      r11,0;                          \
-        std     r10,SOFTE(r1);                  \
+        stb     r11,PACASOFTIRQEN(r13);         \
+BEGIN_FW_FTR_SECTION;                           \
+        stb     r11,PACAHARDIRQEN(r13);         \
+END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES);   \
+BEGIN_FW_FTR_SECTION;                           \
         mfmsr   r10;                            \
-        stb     r11,PACAPROCENABLED(r13);       \
         ori     r10,r10,MSR_EE;                 \
         mtmsrd  r10,1;                          \
 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
 
-#define ENABLE_INTS                             \
-BEGIN_FW_FTR_SECTION;                           \
-        lbz     r10,PACAPROCENABLED(r13);       \
-        mfmsr   r11;                            \
-        std     r10,SOFTE(r1);                  \
-        ori     r11,r11,MSR_EE;                 \
-END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES);   \
-BEGIN_FW_FTR_SECTION;                           \
-        ld      r12,_MSR(r1);                   \
-        mfmsr   r11;                            \
-        rlwimi  r11,r12,0,MSR_EE;               \
-END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES);   \
-        mtmsrd  r11,1
+#else
+#define DISABLE_INTS                            \
+        li      r11,0;                          \
+        stb     r11,PACASOFTIRQEN(r13);         \
+        stb     r11,PACAHARDIRQEN(r13)
 
-#else   /* hard enable/disable interrupts */
-#define DISABLE_INTS
+#endif /* CONFIG_PPC_ISERIES */
 
 #define ENABLE_INTS                             \
         ld      r12,_MSR(r1);                   \
@@ -399,8 +419,6 @@ END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES);	\
         rlwimi  r11,r12,0,MSR_EE;               \
         mtmsrd  r11,1
 
-#endif
-
 #define STD_EXCEPTION_COMMON(trap, label, hdlr)         \
         .align  7;                                      \
         .globl label##_common;                          \
@@ -541,11 +559,11 @@ instruction_access_slb_pSeries:
         mfspr   r12,SPRN_SRR1           /* and SRR1 */
         b       .slb_miss_realmode      /* Rel. branch works in real mode */
 
-        STD_EXCEPTION_PSERIES(0x500, hardware_interrupt)
+        MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt)
         STD_EXCEPTION_PSERIES(0x600, alignment)
         STD_EXCEPTION_PSERIES(0x700, program_check)
         STD_EXCEPTION_PSERIES(0x800, fp_unavailable)
-        STD_EXCEPTION_PSERIES(0x900, decrementer)
+        MASKABLE_EXCEPTION_PSERIES(0x900, decrementer)
         STD_EXCEPTION_PSERIES(0xa00, trap_0a)
         STD_EXCEPTION_PSERIES(0xb00, trap_0b)
 
@@ -597,7 +615,24 @@ system_call_pSeries:
 /*** pSeries interrupt support ***/
 
         /* moved from 0xf00 */
-        STD_EXCEPTION_PSERIES(., performance_monitor)
+        MASKABLE_EXCEPTION_PSERIES(., performance_monitor)
+
+/*
+ * An interrupt came in while soft-disabled; clear EE in SRR1,
+ * clear paca->hard_enabled and return.
+ */
+masked_interrupt:
+        stb     r10,PACAHARDIRQEN(r13)
+        mtcrf   0x80,r9
+        ld      r9,PACA_EXGEN+EX_R9(r13)
+        mfspr   r10,SPRN_SRR1
+        rldicl  r10,r10,48,1            /* clear MSR_EE */
+        rotldi  r10,r10,16
+        mtspr   SPRN_SRR1,r10
+        ld      r10,PACA_EXGEN+EX_R10(r13)
+        mfspr   r13,SPRN_SPRG1
+        rfid
+        b       .
 
         .align  7
 _GLOBAL(do_stab_bolted_pSeries)
@@ -952,7 +987,8 @@ fast_exception_return:
         REST_8GPRS(2, r1)
 
         mfmsr   r10
-        clrrdi  r10,r10,2               /* clear RI (LE is 0 already) */
+        rldicl  r10,r10,48,1            /* clear EE */
+        rldicr  r10,r10,16,61           /* clear RI (LE is 0 already) */
         mtmsrd  r10,1
 
         mtspr   SPRN_SRR1,r12
@@ -1877,11 +1913,16 @@ _GLOBAL(__secondary_start)
         /* enable MMU and jump to start_secondary */
         LOAD_REG_ADDR(r3, .start_secondary_prolog)
         LOAD_REG_IMMEDIATE(r4, MSR_KERNEL)
-#ifdef DO_SOFT_DISABLE
+#ifdef CONFIG_PPC_ISERIES
 BEGIN_FW_FTR_SECTION
         ori     r4,r4,MSR_EE
 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
 #endif
+BEGIN_FW_FTR_SECTION
+        stb     r7,PACASOFTIRQEN(r13)
+        stb     r7,PACAHARDIRQEN(r13)
+END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES)
+
         mtspr   SPRN_SRR0,r3
         mtspr   SPRN_SRR1,r4
         rfid
@@ -2019,15 +2060,18 @@ _STATIC(start_here_common)
 
         /* Load up the kernel context */
 5:
-#ifdef DO_SOFT_DISABLE
-BEGIN_FW_FTR_SECTION
         li      r5,0
-        stb     r5,PACAPROCENABLED(r13) /* Soft Disabled */
+        stb     r5,PACASOFTIRQEN(r13)   /* Soft Disabled */
+#ifdef CONFIG_PPC_ISERIES
+BEGIN_FW_FTR_SECTION
         mfmsr   r5
         ori     r5,r5,MSR_EE            /* Hard Enabled */
         mtmsrd  r5
 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
 #endif
+BEGIN_FW_FTR_SECTION
+        stb     r5,PACAHARDIRQEN(r13)
+END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES)
 
         bl .start_kernel
 
diff --git a/arch/powerpc/kernel/idle_power4.S b/arch/powerpc/kernel/idle_power4.S
index 30de81da7b40..ba3195478600 100644
--- a/arch/powerpc/kernel/idle_power4.S
+++ b/arch/powerpc/kernel/idle_power4.S
@@ -30,6 +30,13 @@ END_FTR_SECTION_IFCLR(CPU_FTR_CAN_NAP)
         beqlr
 
         /* Go to NAP now */
+        mfmsr   r7
+        rldicl  r0,r7,48,1
+        rotldi  r0,r0,16
+        mtmsrd  r0,1                    /* hard-disable interrupts */
+        li      r0,1
+        stb     r0,PACASOFTIRQEN(r13)   /* we'll hard-enable shortly */
+        stb     r0,PACAHARDIRQEN(r13)
 BEGIN_FTR_SECTION
         DSSALL
         sync
@@ -38,7 +45,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
         ld      r8,TI_LOCAL_FLAGS(r9)   /* set napping bit */
         ori     r8,r8,_TLF_NAPPING      /* so when we take an exception */
         std     r8,TI_LOCAL_FLAGS(r9)   /* it will return to our caller */
-        mfmsr   r7
         ori     r7,r7,MSR_EE
         oris    r7,r7,MSR_POW@h
 1:      sync
diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c
index 5e37bf14ef2d..67b21a008f6e 100644
--- a/arch/powerpc/kernel/irq.c
+++ b/arch/powerpc/kernel/irq.c
@@ -64,8 +64,9 @@
 #include <asm/ptrace.h>
 #include <asm/machdep.h>
 #include <asm/udbg.h>
-#ifdef CONFIG_PPC_ISERIES
+#ifdef CONFIG_PPC64
 #include <asm/paca.h>
+#include <asm/firmware.h>
 #endif
 
 int __irq_offset_value;
@@ -95,6 +96,27 @@ extern atomic_t ipi_sent;
 EXPORT_SYMBOL(irq_desc);
 
 int distribute_irqs = 1;
+
+void local_irq_restore(unsigned long en)
+{
+        get_paca()->soft_enabled = en;
+        if (!en)
+                return;
+
+        if (firmware_has_feature(FW_FEATURE_ISERIES)) {
+                if (get_paca()->lppaca_ptr->int_dword.any_int)
+                        iseries_handle_interrupts();
+                return;
+        }
+
+        if (get_paca()->hard_enabled)
+                return;
+        /* need to hard-enable interrupts here */
+        get_paca()->hard_enabled = en;
+        if ((int)mfspr(SPRN_DEC) < 0)
+                mtspr(SPRN_DEC, 1);
+        hard_irq_enable();
+}
 #endif /* CONFIG_PPC64 */
 
 int show_interrupts(struct seq_file *p, void *v)
diff --git a/arch/powerpc/kernel/ppc_ksyms.c b/arch/powerpc/kernel/ppc_ksyms.c
index 807193a3c784..9179f0739ea2 100644
--- a/arch/powerpc/kernel/ppc_ksyms.c
+++ b/arch/powerpc/kernel/ppc_ksyms.c
@@ -49,6 +49,10 @@
 #include <asm/commproc.h>
 #endif
 
+#ifdef CONFIG_PPC64
+EXPORT_SYMBOL(local_irq_restore);
+#endif
+
 #ifdef CONFIG_PPC32
 extern void transfer_to_handler(void);
 extern void do_IRQ(struct pt_regs *regs);
diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c
index 4b2e32eab9dc..b1b0cda3f748 100644
--- a/arch/powerpc/kernel/setup_64.c
+++ b/arch/powerpc/kernel/setup_64.c
@@ -223,8 +223,8 @@ void early_setup_secondary(void)
 {
         struct paca_struct *lpaca = get_paca();
 
-        /* Mark enabled in PACA */
-        lpaca->proc_enabled = 0;
+        /* Mark interrupts enabled in PACA */
+        lpaca->soft_enabled = 0;
 
         /* Initialize hash table for that CPU */
         htab_initialize_secondary();