5 files changed, 187 insertions, 164 deletions

diff --git a/arch/sh/mm/Makefile b/arch/sh/mm/Makefile
index f4e32b3d24dc..d90906367c5f 100644
--- a/arch/sh/mm/Makefile
+++ b/arch/sh/mm/Makefile
@@ -12,7 +12,7 @@ obj-$(CONFIG_DMA_PAGE_OPS)	+= pg-dma.o
 obj-$(CONFIG_HUGETLB_PAGE)      += hugetlbpage.o
 
 mmu-y                   := fault-nommu.o tlb-nommu.o pg-nommu.o
-mmu-$(CONFIG_MMU)       := fault.o clear_page.o copy_page.o
+mmu-$(CONFIG_MMU)       := fault.o clear_page.o copy_page.o tlb-flush.o
 
 obj-y                   += $(mmu-y)
 
diff --git a/arch/sh/mm/consistent.c b/arch/sh/mm/consistent.c
index ee73e30263af..c81e6b67ad30 100644
--- a/arch/sh/mm/consistent.c
+++ b/arch/sh/mm/consistent.c
@@ -9,6 +9,8 @@
  */
 #include <linux/mm.h>
 #include <linux/dma-mapping.h>
+#include <asm/cacheflush.h>
+#include <asm/addrspace.h>
 #include <asm/io.h>
 
 void *consistent_alloc(gfp_t gfp, size_t size, dma_addr_t *handle)
diff --git a/arch/sh/mm/fault.c b/arch/sh/mm/fault.c
index 364181f27b79..7a03ffe6dadd 100644
--- a/arch/sh/mm/fault.c
+++ b/arch/sh/mm/fault.c
@@ -1,33 +1,20 @@
-/* $Id: fault.c,v 1.14 2004/01/13 05:52:11 kkojima Exp $
+/*
+ * Page fault handler for SH with an MMU.
  *
- *  linux/arch/sh/mm/fault.c
  *  Copyright (C) 1999  Niibe Yutaka
  *  Copyright (C) 2003  Paul Mundt
  *
  *  Based on linux/arch/i386/mm/fault.c:
  *   Copyright (C) 1995  Linus Torvalds
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
  */
-
-#include <linux/signal.h>
-#include <linux/sched.h>
 #include <linux/kernel.h>
-#include <linux/errno.h>
-#include <linux/string.h>
-#include <linux/types.h>
-#include <linux/ptrace.h>
-#include <linux/mman.h>
 #include <linux/mm.h>
-#include <linux/smp.h>
-#include <linux/smp_lock.h>
-#include <linux/interrupt.h>
-#include <linux/module.h>
-
 #include <asm/system.h>
-#include <asm/io.h>
-#include <asm/uaccess.h>
-#include <asm/pgalloc.h>
 #include <asm/mmu_context.h>
-#include <asm/cacheflush.h>
 #include <asm/kgdb.h>
 
 extern void die(const char *,struct pt_regs *,long);
@@ -187,14 +174,25 @@ do_sigbus:
                 goto no_context;
 }
 
+#ifdef CONFIG_SH_STORE_QUEUES
 /*
- * Called with interrupt disabled.
+ * This is a special case for the SH-4 store queues, as pages for this
+ * space still need to be faulted in before it's possible to flush the
+ * store queue cache for writeout to the remapped region.
+ */
+#define P3_ADDR_MAX             (P4SEG_STORE_QUE + 0x04000000)
+#else
+#define P3_ADDR_MAX             P4SEG
+#endif
+
+/*
+ * Called with interrupts disabled.
  */
 asmlinkage int __do_page_fault(struct pt_regs *regs, unsigned long writeaccess,
                                unsigned long address)
 {
-        unsigned long addrmax = P4SEG;
         pgd_t *pgd;
+        pud_t *pud;
         pmd_t *pmd;
         pte_t *pte;
         pte_t entry;
@@ -207,31 +205,36 @@ asmlinkage int __do_page_fault(struct pt_regs *regs, unsigned long writeaccess,
                 kgdb_bus_err_hook();
 #endif
 
-#ifdef CONFIG_SH_STORE_QUEUES
-        addrmax = P4SEG_STORE_QUE + 0x04000000;
-#endif
-
-        if (address >= P3SEG && address < addrmax) {
+        /*
+         * We don't take page faults for P1, P2, and parts of P4, these
+         * are always mapped, whether it be due to legacy behaviour in
+         * 29-bit mode, or due to PMB configuration in 32-bit mode.
+         */
+        if (address >= P3SEG && address < P3_ADDR_MAX)
                 pgd = pgd_offset_k(address);
-                mm = NULL;
-        } else if (address >= TASK_SIZE)
-                return 1;
-        else if (!(mm = current->mm))
-                return 1;
-        else
-                pgd = pgd_offset(mm, address);
+        else {
+                if (unlikely(address >= TASK_SIZE || !current->mm))
+                        return 1;
+
+                pgd = pgd_offset(current->mm, address);
+        }
 
-        pmd = pmd_offset(pgd, address);
+        pud = pud_offset(pgd, address);
+        if (pud_none_or_clear_bad(pud))
+                return 1;
+        pmd = pmd_offset(pud, address);
         if (pmd_none_or_clear_bad(pmd))
                 return 1;
+
         if (mm)
                 pte = pte_offset_map_lock(mm, pmd, address, &ptl);
         else
                 pte = pte_offset_kernel(pmd, address);
 
         entry = *pte;
-        if (pte_none(entry) || pte_not_present(entry)
-            || (writeaccess && !pte_write(entry)))
+        if (unlikely(pte_none(entry) || pte_not_present(entry)))
+                goto unlock;
+        if (unlikely(writeaccess && !pte_write(entry)))
                 goto unlock;
 
         if (writeaccess)
@@ -243,13 +246,7 @@ asmlinkage int __do_page_fault(struct pt_regs *regs, unsigned long writeaccess,
          * ITLB is not affected by "ldtlb" instruction.
          * So, we need to flush the entry by ourselves.
          */
-
-        {
-                unsigned long flags;
-                local_irq_save(flags);
-                __flush_tlb_page(get_asid(), address&PAGE_MASK);
-                local_irq_restore(flags);
-        }
+        __flush_tlb_page(get_asid(), address & PAGE_MASK);
 #endif
 
         set_pte(pte, entry);
@@ -260,122 +257,3 @@ unlock:
                 pte_unmap_unlock(pte, ptl);
         return ret;
 }
-
-void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
-{
-        if (vma->vm_mm && vma->vm_mm->context != NO_CONTEXT) {
-                unsigned long flags;
-                unsigned long asid;
-                unsigned long saved_asid = MMU_NO_ASID;
-
-                asid = vma->vm_mm->context & MMU_CONTEXT_ASID_MASK;
-                page &= PAGE_MASK;
-
-                local_irq_save(flags);
-                if (vma->vm_mm != current->mm) {
-                        saved_asid = get_asid();
-                        set_asid(asid);
-                }
-                __flush_tlb_page(asid, page);
-                if (saved_asid != MMU_NO_ASID)
-                        set_asid(saved_asid);
-                local_irq_restore(flags);
-        }
-}
-
-void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
-                     unsigned long end)
-{
-        struct mm_struct *mm = vma->vm_mm;
-
-        if (mm->context != NO_CONTEXT) {
-                unsigned long flags;
-                int size;
-
-                local_irq_save(flags);
-                size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
-                if (size > (MMU_NTLB_ENTRIES/4)) { /* Too many TLB to flush */
-                        mm->context = NO_CONTEXT;
-                        if (mm == current->mm)
-                                activate_context(mm);
-                } else {
-                        unsigned long asid = mm->context&MMU_CONTEXT_ASID_MASK;
-                        unsigned long saved_asid = MMU_NO_ASID;
-
-                        start &= PAGE_MASK;
-                        end += (PAGE_SIZE - 1);
-                        end &= PAGE_MASK;
-                        if (mm != current->mm) {
-                                saved_asid = get_asid();
-                                set_asid(asid);
-                        }
-                        while (start < end) {
-                                __flush_tlb_page(asid, start);
-                                start += PAGE_SIZE;
-                        }
-                        if (saved_asid != MMU_NO_ASID)
-                                set_asid(saved_asid);
-                }
-                local_irq_restore(flags);
-        }
-}
-
-void flush_tlb_kernel_range(unsigned long start, unsigned long end)
-{
-        unsigned long flags;
-        int size;
-
-        local_irq_save(flags);
-        size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
-        if (size > (MMU_NTLB_ENTRIES/4)) { /* Too many TLB to flush */
-                flush_tlb_all();
-        } else {
-                unsigned long asid = init_mm.context&MMU_CONTEXT_ASID_MASK;
-                unsigned long saved_asid = get_asid();
-
-                start &= PAGE_MASK;
-                end += (PAGE_SIZE - 1);
-                end &= PAGE_MASK;
-                set_asid(asid);
-                while (start < end) {
-                        __flush_tlb_page(asid, start);
-                        start += PAGE_SIZE;
-                }
-                set_asid(saved_asid);
-        }
-        local_irq_restore(flags);
-}
-
-void flush_tlb_mm(struct mm_struct *mm)
-{
-        /* Invalidate all TLB of this process. */
-        /* Instead of invalidating each TLB, we get new MMU context. */
-        if (mm->context != NO_CONTEXT) {
-                unsigned long flags;
-
-                local_irq_save(flags);
-                mm->context = NO_CONTEXT;
-                if (mm == current->mm)
-                        activate_context(mm);
-                local_irq_restore(flags);
-        }
-}
-
-void flush_tlb_all(void)
-{
-        unsigned long flags, status;
-
-        /*
-         * Flush all the TLB.
-         *
-         * Write to the MMU control register's bit:
-         *      TF-bit for SH-3, TI-bit for SH-4.
-         *      It's same position, bit #2.
-         */
-        local_irq_save(flags);
-        status = ctrl_inl(MMUCR);
-        status |= 0x04;
-        ctrl_outl(status, MMUCR);
-        ctrl_barrier();
-        local_irq_restore(flags);
-}
diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c
index 8ea27ca4b700..d1a979eab656 100644
--- a/arch/sh/mm/init.c
+++ b/arch/sh/mm/init.c
@@ -80,6 +80,7 @@ void show_mem(void)
 static void set_pte_phys(unsigned long addr, unsigned long phys, pgprot_t prot)
 {
         pgd_t *pgd;
+        pud_t *pud;
         pmd_t *pmd;
         pte_t *pte;
 
@@ -89,7 +90,17 @@ static void set_pte_phys(unsigned long addr, unsigned long phys, pgprot_t prot)
                 return;
         }
 
-        pmd = pmd_offset(pgd, addr);
+        pud = pud_offset(pgd, addr);
+        if (pud_none(*pud)) {
+                pmd = (pmd_t *)get_zeroed_page(GFP_ATOMIC);
+                set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE | _PAGE_USER));
+                if (pmd != pmd_offset(pud, 0)) {
+                        pud_ERROR(*pud);
+                        return;
+                }
+        }
+
+        pmd = pmd_offset(pud, addr);
         if (pmd_none(*pmd)) {
                 pte = (pte_t *)get_zeroed_page(GFP_ATOMIC);
                 set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE | _PAGE_USER));
diff --git a/arch/sh/mm/tlb-flush.c b/arch/sh/mm/tlb-flush.c
new file mode 100644
index 000000000000..fd7e42bcaa40
--- /dev/null
+++ b/arch/sh/mm/tlb-flush.c
@@ -0,0 +1,132 @@
+/*
+ * TLB flushing operations for SH with an MMU.
+ *
+ *  Copyright (C) 1999  Niibe Yutaka
+ *  Copyright (C) 2003  Paul Mundt
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ */
+#include <linux/mm.h>
+#include <asm/mmu_context.h>
+#include <asm/tlbflush.h>
+
+void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
+{
+        if (vma->vm_mm && vma->vm_mm->context != NO_CONTEXT) {
+                unsigned long flags;
+                unsigned long asid;
+                unsigned long saved_asid = MMU_NO_ASID;
+
+                asid = vma->vm_mm->context & MMU_CONTEXT_ASID_MASK;
+                page &= PAGE_MASK;
+
+                local_irq_save(flags);
+                if (vma->vm_mm != current->mm) {
+                        saved_asid = get_asid();
+                        set_asid(asid);
+                }
+                __flush_tlb_page(asid, page);
+                if (saved_asid != MMU_NO_ASID)
+                        set_asid(saved_asid);
+                local_irq_restore(flags);
+        }
+}
+
+void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
+                     unsigned long end)
+{
+        struct mm_struct *mm = vma->vm_mm;
+
+        if (mm->context != NO_CONTEXT) {
+                unsigned long flags;
+                int size;
+
+                local_irq_save(flags);
+                size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
+                if (size > (MMU_NTLB_ENTRIES/4)) { /* Too many TLB to flush */
+                        mm->context = NO_CONTEXT;
+                        if (mm == current->mm)
+                                activate_context(mm);
+                } else {
+                        unsigned long asid = mm->context&MMU_CONTEXT_ASID_MASK;
+                        unsigned long saved_asid = MMU_NO_ASID;
+
+                        start &= PAGE_MASK;
+                        end += (PAGE_SIZE - 1);
+                        end &= PAGE_MASK;
+                        if (mm != current->mm) {
+                                saved_asid = get_asid();
+                                set_asid(asid);
+                        }
+                        while (start < end) {
+                                __flush_tlb_page(asid, start);
+                                start += PAGE_SIZE;
+                        }
+                        if (saved_asid != MMU_NO_ASID)
+                                set_asid(saved_asid);
+                }
+                local_irq_restore(flags);
+        }
+}
+
+void flush_tlb_kernel_range(unsigned long start, unsigned long end)
+{
+        unsigned long flags;
+        int size;
+
+        local_irq_save(flags);
+        size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
+        if (size > (MMU_NTLB_ENTRIES/4)) { /* Too many TLB to flush */
+                flush_tlb_all();
+        } else {
+                unsigned long asid = init_mm.context&MMU_CONTEXT_ASID_MASK;
+                unsigned long saved_asid = get_asid();
+
+                start &= PAGE_MASK;
+                end += (PAGE_SIZE - 1);
+                end &= PAGE_MASK;
+                set_asid(asid);
+                while (start < end) {
+                        __flush_tlb_page(asid, start);
+                        start += PAGE_SIZE;
+                }
+                set_asid(saved_asid);
+        }
+        local_irq_restore(flags);
+}
+
+void flush_tlb_mm(struct mm_struct *mm)
+{
+        /* Invalidate all TLB of this process. */
+        /* Instead of invalidating each TLB, we get new MMU context. */
+        if (mm->context != NO_CONTEXT) {
+                unsigned long flags;
+
+                local_irq_save(flags);
+                mm->context = NO_CONTEXT;
+                if (mm == current->mm)
+                        activate_context(mm);
+                local_irq_restore(flags);
+        }
+}
+
+void flush_tlb_all(void)
+{
+        unsigned long flags, status;
+
+        /*
+         * Flush all the TLB.
+         *
+         * Write to the MMU control register's bit:
+         *      TF-bit for SH-3, TI-bit for SH-4.
+         *      It's same position, bit #2.
+         */
+        local_irq_save(flags);
+        status = ctrl_inl(MMUCR);
+        status |= 0x04;
+        ctrl_outl(status, MMUCR);
+        ctrl_barrier();
+        local_irq_restore(flags);
+}