8 files changed, 116 insertions, 128 deletions

diff --git a/arch/powerpc/mm/44x_mmu.c b/arch/powerpc/mm/44x_mmu.c
index 0a0a0487b334..ca4dcb07a939 100644
--- a/arch/powerpc/mm/44x_mmu.c
+++ b/arch/powerpc/mm/44x_mmu.c
@@ -24,73 +24,38 @@
  *
  */
 
-#include <linux/signal.h>
-#include <linux/sched.h>
-#include <linux/kernel.h>
-#include <linux/errno.h>
-#include <linux/string.h>
-#include <linux/types.h>
-#include <linux/ptrace.h>
-#include <linux/mman.h>
-#include <linux/mm.h>
-#include <linux/swap.h>
-#include <linux/stddef.h>
-#include <linux/vmalloc.h>
 #include <linux/init.h>
-#include <linux/delay.h>
-#include <linux/highmem.h>
-
-#include <asm/pgalloc.h>
-#include <asm/prom.h>
-#include <asm/io.h>
-#include <asm/mmu_context.h>
-#include <asm/pgtable.h>
 #include <asm/mmu.h>
-#include <asm/uaccess.h>
-#include <asm/smp.h>
-#include <asm/bootx.h>
-#include <asm/machdep.h>
-#include <asm/setup.h>
+#include <asm/system.h>
+#include <asm/page.h>
 
 #include "mmu_decl.h"
 
-extern char etext[], _stext[];
-
 /* Used by the 44x TLB replacement exception handler.
  * Just needed it declared someplace.
  */
-unsigned int tlb_44x_index = 0;
-unsigned int tlb_44x_hwater = 62;
+unsigned int tlb_44x_index; /* = 0 */
+unsigned int tlb_44x_hwater = PPC44x_TLB_SIZE - 1 - PPC44x_EARLY_TLBS;
 
 /*
  * "Pins" a 256MB TLB entry in AS0 for kernel lowmem
  */
-static void __init
-ppc44x_pin_tlb(int slot, unsigned int virt, unsigned int phys)
+static void __init ppc44x_pin_tlb(unsigned int virt, unsigned int phys)
 {
-        unsigned long attrib = 0;
-
-        __asm__ __volatile__("\
-        clrrwi  %2,%2,10\n\
-        ori     %2,%2,%4\n\
-        clrrwi  %1,%1,10\n\
-        li      %0,0\n\
-        ori     %0,%0,%5\n\
-        tlbwe   %2,%3,%6\n\
-        tlbwe   %1,%3,%7\n\
-        tlbwe   %0,%3,%8"
+        __asm__ __volatile__(
+                "tlbwe  %2,%3,%4\n"
+                "tlbwe  %1,%3,%5\n"
+                "tlbwe  %0,%3,%6\n"
         :
-        : "r" (attrib), "r" (phys), "r" (virt), "r" (slot),
-          "i" (PPC44x_TLB_VALID | PPC44x_TLB_256M),
-          "i" (PPC44x_TLB_SW | PPC44x_TLB_SR | PPC44x_TLB_SX | PPC44x_TLB_G),
+        : "r" (PPC44x_TLB_SW | PPC44x_TLB_SR | PPC44x_TLB_SX | PPC44x_TLB_G),
+          "r" (phys),
+          "r" (virt | PPC44x_TLB_VALID | PPC44x_TLB_256M),
+          "r" (tlb_44x_hwater--), /* slot for this TLB entry */
           "i" (PPC44x_TLB_PAGEID),
           "i" (PPC44x_TLB_XLAT),
           "i" (PPC44x_TLB_ATTRIB));
 }
 
-/*
- * MMU_init_hw does the chip-specific initialization of the MMU hardware.
- */
 void __init MMU_init_hw(void)
 {
         flush_instruction_cache();
@@ -98,22 +63,13 @@ void __init MMU_init_hw(void)
 
 unsigned long __init mmu_mapin_ram(void)
 {
-        unsigned int pinned_tlbs = 1;
-        int i;
-
-        /* Determine number of entries necessary to cover lowmem */
-        pinned_tlbs = (unsigned int)
-                (_ALIGN(total_lowmem, PPC_PIN_SIZE) >> PPC44x_PIN_SHIFT);
-
-        /* Write upper watermark to save location */
-        tlb_44x_hwater = PPC44x_LOW_SLOT - pinned_tlbs;
+        unsigned long addr;
 
-        /* If necessary, set additional pinned TLBs */
-        if (pinned_tlbs > 1)
-                for (i = (PPC44x_LOW_SLOT-(pinned_tlbs-1)); i < PPC44x_LOW_SLOT; i++) {
-                        unsigned int phys_addr = (PPC44x_LOW_SLOT-i) * PPC_PIN_SIZE;
-                        ppc44x_pin_tlb(i, phys_addr+PAGE_OFFSET, phys_addr);
-                }
+        /* Pin in enough TLBs to cover any lowmem not covered by the
+         * initial 256M mapping established in head_44x.S */
+        for (addr = PPC_PIN_SIZE; addr < total_lowmem;
+             addr += PPC_PIN_SIZE)
+                ppc44x_pin_tlb(addr + PAGE_OFFSET, addr);
 
         return total_lowmem;
 }
diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
index bec0cce79a78..bfe901353142 100644
--- a/arch/powerpc/mm/fault.c
+++ b/arch/powerpc/mm/fault.c
@@ -39,37 +39,26 @@
 #include <asm/tlbflush.h>
 #include <asm/siginfo.h>
 
-#ifdef CONFIG_KPROBES
-ATOMIC_NOTIFIER_HEAD(notify_page_fault_chain);
 
-/* Hook to register for page fault notifications */
-int register_page_fault_notifier(struct notifier_block *nb)
-{
-        return atomic_notifier_chain_register(&notify_page_fault_chain, nb);
-}
-
-int unregister_page_fault_notifier(struct notifier_block *nb)
+#ifdef CONFIG_KPROBES
+static inline int notify_page_fault(struct pt_regs *regs)
 {
-        return atomic_notifier_chain_unregister(&notify_page_fault_chain, nb);
-}
+        int ret = 0;
+
+        /* kprobe_running() needs smp_processor_id() */
+        if (!user_mode(regs)) {
+                preempt_disable();
+                if (kprobe_running() && kprobe_fault_handler(regs, 11))
+                        ret = 1;
+                preempt_enable();
+        }
 
-static inline int notify_page_fault(enum die_val val, const char *str,
-                        struct pt_regs *regs, long err, int trap, int sig)
-{
-        struct die_args args = {
-                .regs = regs,
-                .str = str,
-                .err = err,
-                .trapnr = trap,
-                .signr = sig
-        };
-        return atomic_notifier_call_chain(&notify_page_fault_chain, val, &args);
+        return ret;
 }
 #else
-static inline int notify_page_fault(enum die_val val, const char *str,
-                        struct pt_regs *regs, long err, int trap, int sig)
+static inline int notify_page_fault(struct pt_regs *regs)
 {
-        return NOTIFY_DONE;
+        return 0;
 }
 #endif
 
@@ -175,8 +164,7 @@ int __kprobes do_page_fault(struct pt_regs *regs, unsigned long address,
         is_write = error_code & ESR_DST;
 #endif /* CONFIG_4xx || CONFIG_BOOKE */
 
-        if (notify_page_fault(DIE_PAGE_FAULT, "page_fault", regs, error_code,
-                                11, SIGSEGV) == NOTIFY_STOP)
+        if (notify_page_fault(regs))
                 return 0;
 
         if (trap == 0x300) {
diff --git a/arch/powerpc/mm/hash_native_64.c b/arch/powerpc/mm/hash_native_64.c
index 79aedaf36f2b..7b7fe2d7b9dc 100644
--- a/arch/powerpc/mm/hash_native_64.c
+++ b/arch/powerpc/mm/hash_native_64.c
@@ -26,6 +26,7 @@
 #include <asm/tlb.h>
 #include <asm/cputable.h>
 #include <asm/udbg.h>
+#include <asm/kexec.h>
 
 #ifdef DEBUG_LOW
 #define DBG_LOW(fmt...) udbg_printf(fmt)
@@ -340,31 +341,70 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long va,
         local_irq_restore(flags);
 }
 
-/*
- * XXX This need fixing based on page size. It's only used by
- * native_hpte_clear() for now which needs fixing too so they
- * make a good pair...
- */
-static unsigned long slot2va(unsigned long hpte_v, unsigned long slot)
-{
-        unsigned long avpn = HPTE_V_AVPN_VAL(hpte_v);
-        unsigned long va;
+#define LP_SHIFT        12
+#define LP_BITS         8
+#define LP_MASK(i)      ((0xFF >> (i)) << LP_SHIFT)
 
-        va = avpn << 23;
+static void hpte_decode(hpte_t *hpte, unsigned long slot,
+                        int *psize, unsigned long *va)
+{
+        unsigned long hpte_r = hpte->r;
+        unsigned long hpte_v = hpte->v;
+        unsigned long avpn;
+        int i, size, shift, penc, avpnm_bits;
+
+        if (!(hpte_v & HPTE_V_LARGE))
+                size = MMU_PAGE_4K;
+        else {
+                for (i = 0; i < LP_BITS; i++) {
+                        if ((hpte_r & LP_MASK(i+1)) == LP_MASK(i+1))
+                                break;
+                }
+                penc = LP_MASK(i+1) >> LP_SHIFT;
+                for (size = 0; size < MMU_PAGE_COUNT; size++) {
 
-        if (! (hpte_v & HPTE_V_LARGE)) {
-                unsigned long vpi, pteg;
+                        /* 4K pages are not represented by LP */
+                        if (size == MMU_PAGE_4K)
+                                continue;
 
-                pteg = slot / HPTES_PER_GROUP;
-                if (hpte_v & HPTE_V_SECONDARY)
-                        pteg = ~pteg;
+                        /* valid entries have a shift value */
+                        if (!mmu_psize_defs[size].shift)
+                                continue;
 
-                vpi = ((va >> 28) ^ pteg) & htab_hash_mask;
+                        if (penc == mmu_psize_defs[size].penc)
+                                break;
+                }
+        }
 
-                va |= vpi << PAGE_SHIFT;
+        /*
+         * FIXME, the code below works for 16M, 64K, and 4K pages as these
+         * fall under the p<=23 rules for calculating the virtual address.
+         * In the case of 16M pages, an extra bit is stolen from the AVPN
+         * field to achieve the requisite 24 bits.
+         *
+         * Does not work for 16G pages or 1 TB segments.
+         */
+        shift = mmu_psize_defs[size].shift;
+        if (mmu_psize_defs[size].avpnm)
+                avpnm_bits = __ilog2_u64(mmu_psize_defs[size].avpnm) + 1;
+        else
+                avpnm_bits = 0;
+        if (shift - avpnm_bits <= 23) {
+                avpn = HPTE_V_AVPN_VAL(hpte_v) << 23;
+
+                if (shift < 23) {
+                        unsigned long vpi, pteg;
+
+                        pteg = slot / HPTES_PER_GROUP;
+                        if (hpte_v & HPTE_V_SECONDARY)
+                                pteg = ~pteg;
+                        vpi = ((avpn >> 28) ^ pteg) & htab_hash_mask;
+                        avpn |= (vpi << mmu_psize_defs[size].shift);
+                }
         }
 
-        return va;
+        *va = avpn;
+        *psize = size;
 }
 
 /*
@@ -374,15 +414,14 @@ static unsigned long slot2va(unsigned long hpte_v, unsigned long slot)
  *
  * TODO: add batching support when enabled.  remember, no dynamic memory here,
  * athough there is the control page available...
- *
- * XXX FIXME: 4k only for now !
  */
 static void native_hpte_clear(void)
 {
         unsigned long slot, slots, flags;
         hpte_t *hptep = htab_address;
-        unsigned long hpte_v;
+        unsigned long hpte_v, va;
         unsigned long pteg_count;
+        int psize;
 
         pteg_count = htab_hash_mask + 1;
 
@@ -408,8 +447,9 @@ static void native_hpte_clear(void)
                  * already hold the native_tlbie_lock.
                  */
                 if (hpte_v & HPTE_V_VALID) {
+                        hpte_decode(hptep, slot, &psize, &va);
                         hptep->v = 0;
-                        __tlbie(slot2va(hpte_v, slot), MMU_PAGE_4K);
+                        __tlbie(va, psize);
                 }
         }
 
diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c
index 49618461defb..9b226fa7006f 100644
--- a/arch/powerpc/mm/hash_utils_64.c
+++ b/arch/powerpc/mm/hash_utils_64.c
@@ -103,7 +103,7 @@ int mmu_ci_restrictions;
 #ifdef CONFIG_DEBUG_PAGEALLOC
 static u8 *linear_map_hash_slots;
 static unsigned long linear_map_hash_count;
-static spinlock_t linear_map_hash_lock;
+static DEFINE_SPINLOCK(linear_map_hash_lock);
 #endif /* CONFIG_DEBUG_PAGEALLOC */
 
 /* There are definitions of page sizes arrays to be used when none
diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
index c4bcd7546424..1a6e08f3298f 100644
--- a/arch/powerpc/mm/mem.c
+++ b/arch/powerpc/mm/mem.c
@@ -80,7 +80,6 @@ int page_is_ram(unsigned long pfn)
         return 0;
 #endif
 }
-EXPORT_SYMBOL(page_is_ram);
 
 pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
                               unsigned long size, pgprot_t vma_prot)
diff --git a/arch/powerpc/mm/mmu_decl.h b/arch/powerpc/mm/mmu_decl.h
index 9c4538bb04b0..2558c34eedaa 100644
--- a/arch/powerpc/mm/mmu_decl.h
+++ b/arch/powerpc/mm/mmu_decl.h
@@ -40,7 +40,8 @@ extern int __map_without_bats;
 extern unsigned long ioremap_base;
 extern unsigned int rtas_data, rtas_size;
 
-extern PTE *Hash, *Hash_end;
+struct _PTE;
+extern struct _PTE *Hash, *Hash_end;
 extern unsigned long Hash_size, Hash_mask;
 
 extern unsigned int num_tlbcam_entries;
diff --git a/arch/powerpc/mm/pgtable_32.c b/arch/powerpc/mm/pgtable_32.c
index bca560374927..d8232b7a08f7 100644
--- a/arch/powerpc/mm/pgtable_32.c
+++ b/arch/powerpc/mm/pgtable_32.c
@@ -261,7 +261,7 @@ int map_page(unsigned long va, phys_addr_t pa, int flags)
         int err = -ENOMEM;
 
         /* Use upper 10 bits of VA to index the first level map */
-        pd = pmd_offset(pgd_offset_k(va), va);
+        pd = pmd_offset(pud_offset(pgd_offset_k(va), va), va);
         /* Use middle 10 bits of VA to index the second-level map */
         pg = pte_alloc_kernel(pd, va);
         if (pg != 0) {
@@ -354,23 +354,27 @@ int
 get_pteptr(struct mm_struct *mm, unsigned long addr, pte_t **ptep, pmd_t **pmdp)
 {
         pgd_t   *pgd;
+        pud_t   *pud;
         pmd_t   *pmd;
         pte_t   *pte;
         int     retval = 0;
 
         pgd = pgd_offset(mm, addr & PAGE_MASK);
         if (pgd) {
-                pmd = pmd_offset(pgd, addr & PAGE_MASK);
-                if (pmd_present(*pmd)) {
-                        pte = pte_offset_map(pmd, addr & PAGE_MASK);
-                        if (pte) {
-                                retval = 1;
-                                *ptep = pte;
-                                if (pmdp)
-                                        *pmdp = pmd;
-                                /* XXX caller needs to do pte_unmap, yuck */
-                        }
-                }
+                pud = pud_offset(pgd, addr & PAGE_MASK);
+                if (pud && pud_present(*pud)) {
+                        pmd = pmd_offset(pud, addr & PAGE_MASK);
+                        if (pmd_present(*pmd)) {
+                                pte = pte_offset_map(pmd, addr & PAGE_MASK);
+                                if (pte) {
+                                        retval = 1;
+                                        *ptep = pte;
+                                        if (pmdp)
+                                                *pmdp = pmd;
+                                        /* XXX caller needs to do pte_unmap, yuck */
+                                }
+                        }
+                }
         }
         return(retval);
 }
diff --git a/arch/powerpc/mm/stab.c b/arch/powerpc/mm/stab.c
index eeeacab548e6..132c6bc66ce1 100644
--- a/arch/powerpc/mm/stab.c
+++ b/arch/powerpc/mm/stab.c
@@ -227,7 +227,7 @@ void switch_stab(struct task_struct *tsk, struct mm_struct *mm)
  * the first (bolted) segment, so that do_stab_bolted won't get a
  * recursive segment miss on the segment table itself.
  */
-void stabs_alloc(void)
+void __init stabs_alloc(void)
 {
         int cpu;