powerpc: rework 4xx PTE access and TLB miss

This is some preliminary work to improve TLB management on SW loaded
TLB powerpc platforms. This introduce support for non-atomic PTE
operations in pgtable-ppc32.h and removes write back to the PTE from
the TLB miss handlers. In addition, the DSI interrupt code no longer
tries to fixup write permission, this is left to generic code, and
_PAGE_HWWRITE is gone.

Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: Josh Boyer <jwboyer@linux.vnet.ibm.com>

4 files changed, 134 insertions, 192 deletions

diff --git a/arch/powerpc/kernel/head_44x.S b/arch/powerpc/kernel/head_44x.S
index 2944529e8bf9..f3a1ea9d7fe4 100644
--- a/arch/powerpc/kernel/head_44x.S
+++ b/arch/powerpc/kernel/head_44x.S
@@ -293,119 +293,9 @@ interrupt_base:
         MCHECK_EXCEPTION(0x0210, MachineCheckA, machine_check_exception)
 
         /* Data Storage Interrupt */
-        START_EXCEPTION(DataStorage)
-        mtspr   SPRN_SPRG0, r10         /* Save some working registers */
-        mtspr   SPRN_SPRG1, r11
-        mtspr   SPRN_SPRG4W, r12
-        mtspr   SPRN_SPRG5W, r13
-        mfcr    r11
-        mtspr   SPRN_SPRG7W, r11
-
-        /*
-         * Check if it was a store fault, if not then bail
-         * because a user tried to access a kernel or
-         * read-protected page.  Otherwise, get the
-         * offending address and handle it.
-         */
-        mfspr   r10, SPRN_ESR
-        andis.  r10, r10, ESR_ST@h
-        beq     2f
-
-        mfspr   r10, SPRN_DEAR          /* Get faulting address */
-
-        /* If we are faulting a kernel address, we have to use the
-         * kernel page tables.
-         */
-        lis     r11, PAGE_OFFSET@h
-        cmplw   r10, r11
-        blt+    3f
-        lis     r11, swapper_pg_dir@h
-        ori     r11, r11, swapper_pg_dir@l
-
-        mfspr   r12,SPRN_MMUCR
-        rlwinm  r12,r12,0,0,23          /* Clear TID */
-
-        b       4f
-
-        /* Get the PGD for the current thread */
-3:
-        mfspr   r11,SPRN_SPRG3
-        lwz     r11,PGDIR(r11)
-
-        /* Load PID into MMUCR TID */
-        mfspr   r12,SPRN_MMUCR          /* Get MMUCR */
-        mfspr   r13,SPRN_PID            /* Get PID */
-        rlwimi  r12,r13,0,24,31         /* Set TID */
-
-4:
-        mtspr   SPRN_MMUCR,r12
-
-        rlwinm  r12, r10, 13, 19, 29    /* Compute pgdir/pmd offset */
-        lwzx    r11, r12, r11           /* Get pgd/pmd entry */
-        rlwinm. r12, r11, 0, 0, 20      /* Extract pt base address */
-        beq     2f                      /* Bail if no table */
-
-        rlwimi  r12, r10, 23, 20, 28    /* Compute pte address */
-        lwz     r11, 4(r12)             /* Get pte entry */
-
-        andi.   r13, r11, _PAGE_RW      /* Is it writeable? */
-        beq     2f                      /* Bail if not */
-
-        /* Update 'changed'.
-        */
-        ori     r11, r11, _PAGE_DIRTY|_PAGE_ACCESSED|_PAGE_HWWRITE
-        stw     r11, 4(r12)             /* Update Linux page table */
-
-        li      r13, PPC44x_TLB_SR@l    /* Set SR */
-        rlwimi  r13, r11, 29, 29, 29    /* SX = _PAGE_HWEXEC */
-        rlwimi  r13, r11, 0, 30, 30     /* SW = _PAGE_RW */
-        rlwimi  r13, r11, 29, 28, 28    /* UR = _PAGE_USER */
-        rlwimi  r12, r11, 31, 26, 26    /* (_PAGE_USER>>1)->r12 */
-        rlwimi  r12, r11, 29, 30, 30    /* (_PAGE_USER>>3)->r12 */
-        and     r12, r12, r11           /* HWEXEC/RW & USER */
-        rlwimi  r13, r12, 0, 26, 26     /* UX = HWEXEC & USER */
-        rlwimi  r13, r12, 3, 27, 27     /* UW = RW & USER */
-
-        rlwimi  r11,r13,0,26,31         /* Insert static perms */
-
-        /*
-         * Clear U0-U3 and WL1 IL1I IL1D IL2I IL2D bits which are added
-         * on newer 440 cores like the 440x6 used on AMCC 460EX/460GT (see
-         * include/asm-powerpc/pgtable-ppc32.h for details).
-         */
-        rlwinm  r11,r11,0,20,10
-
-        /* find the TLB index that caused the fault.  It has to be here. */
-        tlbsx   r10, 0, r10
-
-        tlbwe   r11, r10, PPC44x_TLB_ATTRIB     /* Write ATTRIB */
-
-        /* Done...restore registers and get out of here.
-        */
-        mfspr   r11, SPRN_SPRG7R
-        mtcr    r11
-        mfspr   r13, SPRN_SPRG5R
-        mfspr   r12, SPRN_SPRG4R
+        DATA_STORAGE_EXCEPTION
 
-        mfspr   r11, SPRN_SPRG1
-        mfspr   r10, SPRN_SPRG0
-        rfi                     /* Force context change */
-
-2:
-        /*
-         * The bailout.  Restore registers to pre-exception conditions
-         * and call the heavyweights to help us out.
-         */
-        mfspr   r11, SPRN_SPRG7R
-        mtcr    r11
-        mfspr   r13, SPRN_SPRG5R
-        mfspr   r12, SPRN_SPRG4R
-
-        mfspr   r11, SPRN_SPRG1
-        mfspr   r10, SPRN_SPRG0
-        b       data_access
-
-        /* Instruction Storage Interrupt */
+                /* Instruction Storage Interrupt */
         INSTRUCTION_STORAGE_EXCEPTION
 
         /* External Input Interrupt */
@@ -423,7 +313,6 @@ interrupt_base:
 #else
         EXCEPTION(0x2010, FloatingPointUnavailable, unknown_exception, EXC_XFER_EE)
 #endif
-
         /* System Call Interrupt */
         START_EXCEPTION(SystemCall)
         NORMAL_EXCEPTION_PROLOG
@@ -484,18 +373,57 @@ interrupt_base:
 4:
         mtspr   SPRN_MMUCR,r12
 
+        /* Mask of required permission bits. Note that while we
+         * do copy ESR:ST to _PAGE_RW position as trying to write
+         * to an RO page is pretty common, we don't do it with
+         * _PAGE_DIRTY. We could do it, but it's a fairly rare
+         * event so I'd rather take the overhead when it happens
+         * rather than adding an instruction here. We should measure
+         * whether the whole thing is worth it in the first place
+         * as we could avoid loading SPRN_ESR completely in the first
+         * place...
+         *
+         * TODO: Is it worth doing that mfspr & rlwimi in the first
+         *       place or can we save a couple of instructions here ?
+         */
+        mfspr   r12,SPRN_ESR
+        li      r13,_PAGE_PRESENT|_PAGE_ACCESSED
+        rlwimi  r13,r12,10,30,30
+
+        /* Load the PTE */
         rlwinm  r12, r10, 13, 19, 29    /* Compute pgdir/pmd offset */
         lwzx    r11, r12, r11           /* Get pgd/pmd entry */
         rlwinm. r12, r11, 0, 0, 20      /* Extract pt base address */
         beq     2f                      /* Bail if no table */
 
         rlwimi  r12, r10, 23, 20, 28    /* Compute pte address */
-        lwz     r11, 4(r12)             /* Get pte entry */
-        andi.   r13, r11, _PAGE_PRESENT /* Is the page present? */
-        beq     2f                      /* Bail if not present */
+        lwz     r11, 0(r12)             /* Get high word of pte entry */
+        lwz     r12, 4(r12)             /* Get low word of pte entry */
 
-        ori     r11, r11, _PAGE_ACCESSED
-        stw     r11, 4(r12)
+        lis     r10,tlb_44x_index@ha
+
+        andc.   r13,r13,r12             /* Check permission */
+
+        /* Load the next available TLB index */
+        lwz     r13,tlb_44x_index@l(r10)
+
+        bne     2f                      /* Bail if permission mismach */
+
+        /* Increment, rollover, and store TLB index */
+        addi    r13,r13,1
+
+        /* Compare with watermark (instruction gets patched) */
+        .globl tlb_44x_patch_hwater_D
+tlb_44x_patch_hwater_D:
+        cmpwi   0,r13,1                 /* reserve entries */
+        ble     5f
+        li      r13,0
+5:
+        /* Store the next available TLB index */
+        stw     r13,tlb_44x_index@l(r10)
+
+        /* Re-load the faulting address */
+        mfspr   r10,SPRN_DEAR
 
          /* Jump to common tlb load */
         b       finish_tlb_load
@@ -510,7 +438,7 @@ interrupt_base:
         mfspr   r12, SPRN_SPRG4R
         mfspr   r11, SPRN_SPRG1
         mfspr   r10, SPRN_SPRG0
-        b       data_access
+        b       DataStorage
 
         /* Instruction TLB Error Interrupt */
         /*
@@ -554,18 +482,42 @@ interrupt_base:
 4:
         mtspr   SPRN_MMUCR,r12
 
+        /* Make up the required permissions */
+        li      r13,_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_HWEXEC
+
         rlwinm  r12, r10, 13, 19, 29    /* Compute pgdir/pmd offset */
         lwzx    r11, r12, r11           /* Get pgd/pmd entry */
         rlwinm. r12, r11, 0, 0, 20      /* Extract pt base address */
         beq     2f                      /* Bail if no table */
 
         rlwimi  r12, r10, 23, 20, 28    /* Compute pte address */
-        lwz     r11, 4(r12)             /* Get pte entry */
-        andi.   r13, r11, _PAGE_PRESENT /* Is the page present? */
-        beq     2f                      /* Bail if not present */
+        lwz     r11, 0(r12)             /* Get high word of pte entry */
+        lwz     r12, 4(r12)             /* Get low word of pte entry */
 
-        ori     r11, r11, _PAGE_ACCESSED
-        stw     r11, 4(r12)
+        lis     r10,tlb_44x_index@ha
+
+        andc.   r13,r13,r12             /* Check permission */
+
+        /* Load the next available TLB index */
+        lwz     r13,tlb_44x_index@l(r10)
+
+        bne     2f                      /* Bail if permission mismach */
+
+        /* Increment, rollover, and store TLB index */
+        addi    r13,r13,1
+
+        /* Compare with watermark (instruction gets patched) */
+        .globl tlb_44x_patch_hwater_I
+tlb_44x_patch_hwater_I:
+        cmpwi   0,r13,1                 /* reserve entries */
+        ble     5f
+        li      r13,0
+5:
+        /* Store the next available TLB index */
+        stw     r13,tlb_44x_index@l(r10)
+
+        /* Re-load the faulting address */
+        mfspr   r10,SPRN_SRR0
 
         /* Jump to common TLB load point */
         b       finish_tlb_load
@@ -587,86 +539,40 @@ interrupt_base:
 
 /*
  * Local functions
- */
-        /*
-         * Data TLB exceptions will bail out to this point
-         * if they can't resolve the lightweight TLB fault.
-         */
-data_access:
-        NORMAL_EXCEPTION_PROLOG
-        mfspr   r5,SPRN_ESR             /* Grab the ESR, save it, pass arg3 */
-        stw     r5,_ESR(r11)
-        mfspr   r4,SPRN_DEAR            /* Grab the DEAR, save it, pass arg2 */
-        EXC_XFER_EE_LITE(0x0300, handle_page_fault)
+  */
 
 /*
 
  * Both the instruction and data TLB miss get to this
  * point to load the TLB.
  *      r10 - EA of fault
- *      r11 - available to use
- *      r12 - Pointer to the 64-bit PTE
- *      r13 - available to use
+ *      r11 - PTE high word value
+ *      r12 - PTE low word value
+ *      r13 - TLB index
  *      MMUCR - loaded with proper value when we get here
  *      Upon exit, we reload everything and RFI.
  */
 finish_tlb_load:
-        /*
-         * We set execute, because we don't have the granularity to
-         * properly set this at the page level (Linux problem).
-         * If shared is set, we cause a zero PID->TID load.
-         * Many of these bits are software only.  Bits we don't set
-         * here we (properly should) assume have the appropriate value.
-         */
-
-        /* Load the next available TLB index */
-        lis     r13, tlb_44x_index@ha
-        lwz     r13, tlb_44x_index@l(r13)
-        /* Load the TLB high watermark */
-        lis     r11, tlb_44x_hwater@ha
-        lwz     r11, tlb_44x_hwater@l(r11)
-
-        /* Increment, rollover, and store TLB index */
-        addi    r13, r13, 1
-        cmpw    0, r13, r11                     /* reserve entries */
-        ble     7f
-        li      r13, 0
-7:
-        /* Store the next available TLB index */
-        lis     r11, tlb_44x_index@ha
-        stw     r13, tlb_44x_index@l(r11)
-
-        lwz     r11, 0(r12)                     /* Get MS word of PTE */
-        lwz     r12, 4(r12)                     /* Get LS word of PTE */
-        rlwimi  r11, r12, 0, 0 , 19             /* Insert RPN */
-        tlbwe   r11, r13, PPC44x_TLB_XLAT       /* Write XLAT */
+        /* Combine RPN & ERPN an write WS 0 */
+        rlwimi  r11,r12,0,0,19
+        tlbwe   r11,r13,PPC44x_TLB_XLAT
 
         /*
-         * Create PAGEID. This is the faulting address,
+         * Create WS1. This is the faulting address (EPN),
          * page size, and valid flag.
          */
-        li      r11, PPC44x_TLB_VALID | PPC44x_TLB_4K
-        rlwimi  r10, r11, 0, 20, 31             /* Insert valid and page size */
-        tlbwe   r10, r13, PPC44x_TLB_PAGEID     /* Write PAGEID */
-
-        li      r10, PPC44x_TLB_SR@l            /* Set SR */
-        rlwimi  r10, r12, 0, 30, 30             /* Set SW = _PAGE_RW */
-        rlwimi  r10, r12, 29, 29, 29            /* SX = _PAGE_HWEXEC */
-        rlwimi  r10, r12, 29, 28, 28            /* UR = _PAGE_USER */
-        rlwimi  r11, r12, 31, 26, 26            /* (_PAGE_USER>>1)->r12 */
-        and     r11, r12, r11                   /* HWEXEC & USER */
-        rlwimi  r10, r11, 0, 26, 26             /* UX = HWEXEC & USER */
-
-        rlwimi  r12, r10, 0, 26, 31             /* Insert static perms */
-
-        /*
-         * Clear U0-U3 and WL1 IL1I IL1D IL2I IL2D bits which are added
-         * on newer 440 cores like the 440x6 used on AMCC 460EX/460GT (see
-         * include/asm-powerpc/pgtable-ppc32.h for details).
-         */
-        rlwinm  r12, r12, 0, 20, 10
-
-        tlbwe   r12, r13, PPC44x_TLB_ATTRIB     /* Write ATTRIB */
+        li      r11,PPC44x_TLB_VALID | PPC44x_TLB_4K
+        rlwimi  r10,r11,0,20,31                 /* Insert valid and page size*/
+        tlbwe   r10,r13,PPC44x_TLB_PAGEID       /* Write PAGEID */
+
+        /* And WS 2 */
+        li      r10,0xf85                       /* Mask to apply from PTE */
+        rlwimi  r10,r12,29,30,30                /* DIRTY -> SW position */
+        and     r11,r12,r10                     /* Mask PTE bits to keep */
+        andi.   r10,r12,_PAGE_USER              /* User page ? */
+        beq     1f                              /* nope, leave U bits empty */
+        rlwimi  r11,r11,3,26,28                 /* yes, copy S bits to U */
+1:      tlbwe   r11,r13,PPC44x_TLB_ATTRIB       /* Write ATTRIB */
 
         /* Done...restore registers and get out of here.
         */
diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h
index b0461be1c928..fce2df988504 100644
--- a/arch/powerpc/kernel/head_booke.h
+++ b/arch/powerpc/kernel/head_booke.h
@@ -340,6 +340,14 @@ label:
         addi    r3,r1,STACK_FRAME_OVERHEAD;                                   \
         EXC_XFER_TEMPLATE(DebugException, 0x2002, (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), NOCOPY, crit_transfer_to_handler, ret_from_crit_exc)
 
+#define DATA_STORAGE_EXCEPTION                                                \
+        START_EXCEPTION(DataStorage)                                          \
+        NORMAL_EXCEPTION_PROLOG;                                              \
+        mfspr   r5,SPRN_ESR;            /* Grab the ESR and save it */        \
+        stw     r5,_ESR(r11);                                                 \
+        mfspr   r4,SPRN_DEAR;           /* Grab the DEAR */                   \
+        EXC_XFER_EE_LITE(0x0300, handle_page_fault)
+
 #define INSTRUCTION_STORAGE_EXCEPTION                                         \
         START_EXCEPTION(InstructionStorage)                                   \
         NORMAL_EXCEPTION_PROLOG;                                              \
diff --git a/arch/powerpc/mm/44x_mmu.c b/arch/powerpc/mm/44x_mmu.c
index 953fb919eb06..98052ac96580 100644
--- a/arch/powerpc/mm/44x_mmu.c
+++ b/arch/powerpc/mm/44x_mmu.c
@@ -27,6 +27,7 @@
 #include <asm/mmu.h>
 #include <asm/system.h>
 #include <asm/page.h>
+#include <asm/cacheflush.h>
 
 #include "mmu_decl.h"
 
@@ -37,11 +38,35 @@ unsigned int tlb_44x_index; /* = 0 */
 unsigned int tlb_44x_hwater = PPC44x_TLB_SIZE - 1 - PPC44x_EARLY_TLBS;
 int icache_44x_need_flush;
 
+static void __init ppc44x_update_tlb_hwater(void)
+{
+        extern unsigned int tlb_44x_patch_hwater_D[];
+        extern unsigned int tlb_44x_patch_hwater_I[];
+
+        /* The TLB miss handlers hard codes the watermark in a cmpli
+         * instruction to improve performances rather than loading it
+         * from the global variable. Thus, we patch the instructions
+         * in the 2 TLB miss handlers when updating the value
+         */
+        tlb_44x_patch_hwater_D[0] = (tlb_44x_patch_hwater_D[0] & 0xffff0000) |
+                tlb_44x_hwater;
+        flush_icache_range((unsigned long)&tlb_44x_patch_hwater_D[0],
+                           (unsigned long)&tlb_44x_patch_hwater_D[1]);
+        tlb_44x_patch_hwater_I[0] = (tlb_44x_patch_hwater_I[0] & 0xffff0000) |
+                tlb_44x_hwater;
+        flush_icache_range((unsigned long)&tlb_44x_patch_hwater_I[0],
+                           (unsigned long)&tlb_44x_patch_hwater_I[1]);
+}
+
 /*
  * "Pins" a 256MB TLB entry in AS0 for kernel lowmem
  */
 static void __init ppc44x_pin_tlb(unsigned int virt, unsigned int phys)
 {
+        unsigned int entry = tlb_44x_hwater--;
+
+        ppc44x_update_tlb_hwater();
+
         __asm__ __volatile__(
                 "tlbwe  %2,%3,%4\n"
                 "tlbwe  %1,%3,%5\n"
@@ -50,7 +75,7 @@ static void __init ppc44x_pin_tlb(unsigned int virt, unsigned int phys)
         : "r" (PPC44x_TLB_SW | PPC44x_TLB_SR | PPC44x_TLB_SX | PPC44x_TLB_G),
           "r" (phys),
           "r" (virt | PPC44x_TLB_VALID | PPC44x_TLB_256M),
-          "r" (tlb_44x_hwater--), /* slot for this TLB entry */
+          "r" (entry),
           "i" (PPC44x_TLB_PAGEID),
           "i" (PPC44x_TLB_XLAT),
           "i" (PPC44x_TLB_ATTRIB));
@@ -58,6 +83,8 @@ static void __init ppc44x_pin_tlb(unsigned int virt, unsigned int phys)
 
 void __init MMU_init_hw(void)
 {
+        ppc44x_update_tlb_hwater();
+
         flush_instruction_cache();
 }
 
diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
index 7b2510799266..1707d00331fc 100644
--- a/arch/powerpc/mm/fault.c
+++ b/arch/powerpc/mm/fault.c
@@ -306,7 +306,8 @@ good_area:
                                         flush_dcache_icache_page(page);
                                         set_bit(PG_arch_1, &page->flags);
                                 }
-                                pte_update(ptep, 0, _PAGE_HWEXEC);
+                                pte_update(ptep, 0, _PAGE_HWEXEC |
+                                           _PAGE_ACCESSED);
                                 _tlbie(address, mm->context.id);
                                 pte_unmap_unlock(ptep, ptl);
                                 up_read(&mm->mmap_sem);