[POWERPC] Lazy interrupt disabling for 64-bit machines

This implements a lazy strategy for disabling interrupts.  This means
that local_irq_disable() et al. just clear the 'interrupts are
enabled' flag in the paca.  If an interrupt comes along, the interrupt
entry code notices that interrupts are supposed to be disabled, and
clears the EE bit in SRR1, clears the 'interrupts are hard-enabled'
flag in the paca, and returns.  This means that interrupts only
actually get disabled in the processor when an interrupt comes along.

When interrupts are enabled by local_irq_enable() et al., the code
sets the interrupts-enabled flag in the paca, and then checks whether
interrupts got hard-disabled.  If so, it also sets the EE bit in the
MSR to hard-enable the interrupts.

This has the potential to improve performance, and also makes it
easier to make a kernel that can boot on iSeries and on other 64-bit
machines, since this lazy-disable strategy is very similar to the
soft-disable strategy that iSeries already uses.

This version renames paca->proc_enabled to paca->soft_enabled, and
changes a couple of soft-disables in the kexec code to hard-disables,
which should fix the crash that Michael Ellerman saw.  This doesn't
yet use a reserved CR field for the soft_enabled and hard_enabled
flags.  This applies on top of Stephen Rothwell's patches to make it
possible to build a combined iSeries/other kernel.

Signed-off-by: Paul Mackerras <paulus@samba.org>

1 files changed, 77 insertions, 33 deletions

diff --git a/arch/powerpc/kernel/head_64.S b/arch/powerpc/kernel/head_64.S
index 645c7f10fb28..c93d9f35a121 100644
--- a/arch/powerpc/kernel/head_64.S
+++ b/arch/powerpc/kernel/head_64.S
@@ -35,9 +35,7 @@
 #include <asm/thread_info.h>
 #include <asm/firmware.h>
 
-#ifdef CONFIG_PPC_ISERIES
 #define DO_SOFT_DISABLE
-#endif
 
 /*
  * We layout physical memory as follows:
@@ -308,7 +306,9 @@ exception_marker:
         std     r9,_LINK(r1);                                              \
         mfctr   r10;                    /* save CTR in stackframe       */ \
         std     r10,_CTR(r1);                                              \
+        lbz     r10,PACASOFTIRQEN(r13);                            \
         mfspr   r11,SPRN_XER;           /* save XER in stackframe       */ \
+        std     r10,SOFTE(r1);                                             \
         std     r11,_XER(r1);                                              \
         li      r9,(n)+1;                                                  \
         std     r9,_TRAP(r1);           /* set trap number              */ \
@@ -343,6 +343,34 @@ label##_pSeries:					\
         EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common)
 
 
+#define MASKABLE_EXCEPTION_PSERIES(n, label)                            \
+        . = n;                                                          \
+        .globl label##_pSeries;                                         \
+label##_pSeries:                                                        \
+        HMT_MEDIUM;                                                     \
+        mtspr   SPRN_SPRG1,r13;         /* save r13 */                  \
+        mfspr   r13,SPRN_SPRG3;         /* get paca address into r13 */ \
+        std     r9,PACA_EXGEN+EX_R9(r13);       /* save r9, r10 */      \
+        std     r10,PACA_EXGEN+EX_R10(r13);                             \
+        lbz     r10,PACASOFTIRQEN(r13);                                 \
+        mfcr    r9;                                                     \
+        cmpwi   r10,0;                                                  \
+        beq     masked_interrupt;                                       \
+        mfspr   r10,SPRN_SPRG1;                                         \
+        std     r10,PACA_EXGEN+EX_R13(r13);                             \
+        std     r11,PACA_EXGEN+EX_R11(r13);                             \
+        std     r12,PACA_EXGEN+EX_R12(r13);                             \
+        clrrdi  r12,r13,32;             /* get high part of &label */   \
+        mfmsr   r10;                                                    \
+        mfspr   r11,SPRN_SRR0;          /* save SRR0 */                 \
+        LOAD_HANDLER(r12,label##_common)                                \
+        ori     r10,r10,MSR_IR|MSR_DR|MSR_RI;                           \
+        mtspr   SPRN_SRR0,r12;                                          \
+        mfspr   r12,SPRN_SRR1;          /* and SRR1 */                  \
+        mtspr   SPRN_SRR1,r10;                                          \
+        rfid;                                                           \
+        b       .       /* prevent speculative execution */
+
 #define STD_EXCEPTION_ISERIES(n, label, area)           \
         .globl label##_iSeries;                         \
 label##_iSeries:                                        \
@@ -358,40 +386,32 @@ label##_iSeries:							\
         HMT_MEDIUM;                                                     \
         mtspr   SPRN_SPRG1,r13;         /* save r13 */                  \
         EXCEPTION_PROLOG_ISERIES_1(PACA_EXGEN);                         \
-        lbz     r10,PACAPROCENABLED(r13);                               \
+        lbz     r10,PACASOFTIRQEN(r13);                                 \
         cmpwi   0,r10,0;                                                \
         beq-    label##_iSeries_masked;                                 \
         EXCEPTION_PROLOG_ISERIES_2;                                     \
         b       label##_common;                                         \
 
-#ifdef DO_SOFT_DISABLE
+#ifdef CONFIG_PPC_ISERIES
 #define DISABLE_INTS                            \
-BEGIN_FW_FTR_SECTION;                           \
-        lbz     r10,PACAPROCENABLED(r13);       \
         li      r11,0;                          \
-        std     r10,SOFTE(r1);                  \
+        stb     r11,PACASOFTIRQEN(r13);         \
+BEGIN_FW_FTR_SECTION;                           \
+        stb     r11,PACAHARDIRQEN(r13);         \
+END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES);   \
+BEGIN_FW_FTR_SECTION;                           \
         mfmsr   r10;                            \
-        stb     r11,PACAPROCENABLED(r13);       \
         ori     r10,r10,MSR_EE;                 \
         mtmsrd  r10,1;                          \
 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
 
-#define ENABLE_INTS                             \
-BEGIN_FW_FTR_SECTION;                           \
-        lbz     r10,PACAPROCENABLED(r13);       \
-        mfmsr   r11;                            \
-        std     r10,SOFTE(r1);                  \
-        ori     r11,r11,MSR_EE;                 \
-END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES);   \
-BEGIN_FW_FTR_SECTION;                           \
-        ld      r12,_MSR(r1);                   \
-        mfmsr   r11;                            \
-        rlwimi  r11,r12,0,MSR_EE;               \
-END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES);   \
-        mtmsrd  r11,1
+#else
+#define DISABLE_INTS                            \
+        li      r11,0;                          \
+        stb     r11,PACASOFTIRQEN(r13);         \
+        stb     r11,PACAHARDIRQEN(r13)
 
-#else   /* hard enable/disable interrupts */
-#define DISABLE_INTS
+#endif /* CONFIG_PPC_ISERIES */
 
 #define ENABLE_INTS                             \
         ld      r12,_MSR(r1);                   \
@@ -399,8 +419,6 @@ END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES);	\
         rlwimi  r11,r12,0,MSR_EE;               \
         mtmsrd  r11,1
 
-#endif
-
 #define STD_EXCEPTION_COMMON(trap, label, hdlr)         \
         .align  7;                                      \
         .globl label##_common;                          \
@@ -541,11 +559,11 @@ instruction_access_slb_pSeries:
         mfspr   r12,SPRN_SRR1           /* and SRR1 */
         b       .slb_miss_realmode      /* Rel. branch works in real mode */
 
-        STD_EXCEPTION_PSERIES(0x500, hardware_interrupt)
+        MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt)
         STD_EXCEPTION_PSERIES(0x600, alignment)
         STD_EXCEPTION_PSERIES(0x700, program_check)
         STD_EXCEPTION_PSERIES(0x800, fp_unavailable)
-        STD_EXCEPTION_PSERIES(0x900, decrementer)
+        MASKABLE_EXCEPTION_PSERIES(0x900, decrementer)
         STD_EXCEPTION_PSERIES(0xa00, trap_0a)
         STD_EXCEPTION_PSERIES(0xb00, trap_0b)
 
@@ -597,7 +615,24 @@ system_call_pSeries:
 /*** pSeries interrupt support ***/
 
         /* moved from 0xf00 */
-        STD_EXCEPTION_PSERIES(., performance_monitor)
+        MASKABLE_EXCEPTION_PSERIES(., performance_monitor)
+
+/*
+ * An interrupt came in while soft-disabled; clear EE in SRR1,
+ * clear paca->hard_enabled and return.
+ */
+masked_interrupt:
+        stb     r10,PACAHARDIRQEN(r13)
+        mtcrf   0x80,r9
+        ld      r9,PACA_EXGEN+EX_R9(r13)
+        mfspr   r10,SPRN_SRR1
+        rldicl  r10,r10,48,1            /* clear MSR_EE */
+        rotldi  r10,r10,16
+        mtspr   SPRN_SRR1,r10
+        ld      r10,PACA_EXGEN+EX_R10(r13)
+        mfspr   r13,SPRN_SPRG1
+        rfid
+        b       .
 
         .align  7
 _GLOBAL(do_stab_bolted_pSeries)
@@ -952,7 +987,8 @@ fast_exception_return:
         REST_8GPRS(2, r1)
 
         mfmsr   r10
-        clrrdi  r10,r10,2               /* clear RI (LE is 0 already) */
+        rldicl  r10,r10,48,1            /* clear EE */
+        rldicr  r10,r10,16,61           /* clear RI (LE is 0 already) */
         mtmsrd  r10,1
 
         mtspr   SPRN_SRR1,r12
@@ -1877,11 +1913,16 @@ _GLOBAL(__secondary_start)
         /* enable MMU and jump to start_secondary */
         LOAD_REG_ADDR(r3, .start_secondary_prolog)
         LOAD_REG_IMMEDIATE(r4, MSR_KERNEL)
-#ifdef DO_SOFT_DISABLE
+#ifdef CONFIG_PPC_ISERIES
 BEGIN_FW_FTR_SECTION
         ori     r4,r4,MSR_EE
 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
 #endif
+BEGIN_FW_FTR_SECTION
+        stb     r7,PACASOFTIRQEN(r13)
+        stb     r7,PACAHARDIRQEN(r13)
+END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES)
+
         mtspr   SPRN_SRR0,r3
         mtspr   SPRN_SRR1,r4
         rfid
@@ -2019,15 +2060,18 @@ _STATIC(start_here_common)
 
         /* Load up the kernel context */
 5:
-#ifdef DO_SOFT_DISABLE
-BEGIN_FW_FTR_SECTION
         li      r5,0
-        stb     r5,PACAPROCENABLED(r13) /* Soft Disabled */
+        stb     r5,PACASOFTIRQEN(r13)   /* Soft Disabled */
+#ifdef CONFIG_PPC_ISERIES
+BEGIN_FW_FTR_SECTION
         mfmsr   r5
         ori     r5,r5,MSR_EE            /* Hard Enabled */
         mtmsrd  r5
 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
 #endif
+BEGIN_FW_FTR_SECTION
+        stb     r5,PACAHARDIRQEN(r13)
+END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES)
 
         bl .start_kernel