1 files changed, 130 insertions, 90 deletions

diff --git a/arch/sh/mm/cache-sh4.c b/arch/sh/mm/cache-sh4.c
index 2203bd6aadb3..aa4f62f0e374 100644
--- a/arch/sh/mm/cache-sh4.c
+++ b/arch/sh/mm/cache-sh4.c
@@ -2,29 +2,31 @@
  * arch/sh/mm/cache-sh4.c
  *
  * Copyright (C) 1999, 2000, 2002  Niibe Yutaka
- * Copyright (C) 2001, 2002, 2003, 2004, 2005  Paul Mundt
+ * Copyright (C) 2001 - 2006  Paul Mundt
  * Copyright (C) 2003  Richard Curnow
  *
  * This file is subject to the terms and conditions of the GNU General Public
  * License.  See the file "COPYING" in the main directory of this archive
  * for more details.
  */
-
 #include <linux/init.h>
-#include <linux/mman.h>
 #include <linux/mm.h>
-#include <linux/threads.h>
 #include <asm/addrspace.h>
-#include <asm/page.h>
 #include <asm/pgtable.h>
 #include <asm/processor.h>
 #include <asm/cache.h>
 #include <asm/io.h>
-#include <asm/uaccess.h>
 #include <asm/pgalloc.h>
 #include <asm/mmu_context.h>
 #include <asm/cacheflush.h>
 
+/*
+ * The maximum number of pages we support up to when doing ranged dcache
+ * flushing. Anything exceeding this will simply flush the dcache in its
+ * entirety.
+ */
+#define MAX_DCACHE_PAGES        64      /* XXX: Tune for ways */
+
 static void __flush_dcache_segment_1way(unsigned long start,
                                         unsigned long extent);
 static void __flush_dcache_segment_2way(unsigned long start,
@@ -219,14 +221,14 @@ void flush_cache_sigtramp(unsigned long addr)
 static inline void flush_cache_4096(unsigned long start,
                                     unsigned long phys)
 {
-        unsigned long flags;
-
         /*
          * All types of SH-4 require PC to be in P2 to operate on the I-cache.
          * Some types of SH-4 require PC to be in P2 to operate on the D-cache.
          */
-        if ((cpu_data->flags & CPU_HAS_P2_FLUSH_BUG)
-           || start < CACHE_OC_ADDRESS_ARRAY) {
+        if ((cpu_data->flags & CPU_HAS_P2_FLUSH_BUG) ||
+            (start < CACHE_OC_ADDRESS_ARRAY)) {
+                unsigned long flags;
+
                 local_irq_save(flags);
                 __flush_cache_4096(start | SH_CACHE_ASSOC,
                                    P1SEGADDR(phys), 0x20000000);
@@ -257,6 +259,7 @@ void flush_dcache_page(struct page *page)
         wmb();
 }
 
+/* TODO: Selective icache invalidation through IC address array.. */
 static inline void flush_icache_all(void)
 {
         unsigned long flags, ccr;
@@ -290,19 +293,121 @@ void flush_cache_all(void)
         flush_icache_all();
 }
 
+static void __flush_cache_mm(struct mm_struct *mm, unsigned long start,
+                             unsigned long end)
+{
+        unsigned long d = 0, p = start & PAGE_MASK;
+        unsigned long alias_mask = cpu_data->dcache.alias_mask;
+        unsigned long n_aliases = cpu_data->dcache.n_aliases;
+        unsigned long select_bit;
+        unsigned long all_aliases_mask;
+        unsigned long addr_offset;
+        pgd_t *dir;
+        pmd_t *pmd;
+        pud_t *pud;
+        pte_t *pte;
+        int i;
+
+        dir = pgd_offset(mm, p);
+        pud = pud_offset(dir, p);
+        pmd = pmd_offset(pud, p);
+        end = PAGE_ALIGN(end);
+
+        all_aliases_mask = (1 << n_aliases) - 1;
+
+        do {
+                if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd))) {
+                        p &= PMD_MASK;
+                        p += PMD_SIZE;
+                        pmd++;
+
+                        continue;
+                }
+
+                pte = pte_offset_kernel(pmd, p);
+
+                do {
+                        unsigned long phys;
+                        pte_t entry = *pte;
+
+                        if (!(pte_val(entry) & _PAGE_PRESENT)) {
+                                pte++;
+                                p += PAGE_SIZE;
+                                continue;
+                        }
+
+                        phys = pte_val(entry) & PTE_PHYS_MASK;
+
+                        if ((p ^ phys) & alias_mask) {
+                                d |= 1 << ((p & alias_mask) >> PAGE_SHIFT);
+                                d |= 1 << ((phys & alias_mask) >> PAGE_SHIFT);
+
+                                if (d == all_aliases_mask)
+                                        goto loop_exit;
+                        }
+
+                        pte++;
+                        p += PAGE_SIZE;
+                } while (p < end && ((unsigned long)pte & ~PAGE_MASK));
+                pmd++;
+        } while (p < end);
+
+loop_exit:
+        addr_offset = 0;
+        select_bit = 1;
+
+        for (i = 0; i < n_aliases; i++) {
+                if (d & select_bit) {
+                        (*__flush_dcache_segment_fn)(addr_offset, PAGE_SIZE);
+                        wmb();
+                }
+
+                select_bit <<= 1;
+                addr_offset += PAGE_SIZE;
+        }
+}
+
+/*
+ * Note : (RPC) since the caches are physically tagged, the only point
+ * of flush_cache_mm for SH-4 is to get rid of aliases from the
+ * D-cache.  The assumption elsewhere, e.g. flush_cache_range, is that
+ * lines can stay resident so long as the virtual address they were
+ * accessed with (hence cache set) is in accord with the physical
+ * address (i.e. tag).  It's no different here.  So I reckon we don't
+ * need to flush the I-cache, since aliases don't matter for that.  We
+ * should try that.
+ *
+ * Caller takes mm->mmap_sem.
+ */
 void flush_cache_mm(struct mm_struct *mm)
 {
         /*
-         * Note : (RPC) since the caches are physically tagged, the only point
-         * of flush_cache_mm for SH-4 is to get rid of aliases from the
-         * D-cache.  The assumption elsewhere, e.g. flush_cache_range, is that
-         * lines can stay resident so long as the virtual address they were
-         * accessed with (hence cache set) is in accord with the physical
-         * address (i.e. tag).  It's no different here.  So I reckon we don't
-         * need to flush the I-cache, since aliases don't matter for that.  We
-         * should try that.
+         * If cache is only 4k-per-way, there are never any 'aliases'.  Since
+         * the cache is physically tagged, the data can just be left in there.
+         */
+        if (cpu_data->dcache.n_aliases == 0)
+                return;
+
+        /*
+         * Don't bother groveling around the dcache for the VMA ranges
+         * if there are too many PTEs to make it worthwhile.
          */
-        flush_cache_all();
+        if (mm->nr_ptes >= MAX_DCACHE_PAGES)
+                flush_dcache_all();
+        else {
+                struct vm_area_struct *vma;
+
+                /*
+                 * In this case there are reasonably sized ranges to flush,
+                 * iterate through the VMA list and take care of any aliases.
+                 */
+                for (vma = mm->mmap; vma; vma = vma->vm_next)
+                        __flush_cache_mm(mm, vma->vm_start, vma->vm_end);
+        }
+
+        /* Only touch the icache if one of the VMAs has VM_EXEC set. */
+        if (mm->exec_vm)
+                flush_icache_all();
 }
 
 /*
@@ -311,7 +416,8 @@ void flush_cache_mm(struct mm_struct *mm)
  * ADDR: Virtual Address (U0 address)
  * PFN: Physical page number
  */
-void flush_cache_page(struct vm_area_struct *vma, unsigned long address, unsigned long pfn)
+void flush_cache_page(struct vm_area_struct *vma, unsigned long address,
+                      unsigned long pfn)
 {
         unsigned long phys = pfn << PAGE_SHIFT;
         unsigned int alias_mask;
@@ -358,87 +464,22 @@ void flush_cache_page(struct vm_area_struct *vma, unsigned long address, unsigne
 void flush_cache_range(struct vm_area_struct *vma, unsigned long start,
                        unsigned long end)
 {
-        unsigned long d = 0, p = start & PAGE_MASK;
-        unsigned long alias_mask = cpu_data->dcache.alias_mask;
-        unsigned long n_aliases = cpu_data->dcache.n_aliases;
-        unsigned long select_bit;
-        unsigned long all_aliases_mask;
-        unsigned long addr_offset;
-        unsigned long phys;
-        pgd_t *dir;
-        pmd_t *pmd;
-        pud_t *pud;
-        pte_t *pte;
-        pte_t entry;
-        int i;
-
         /*
          * If cache is only 4k-per-way, there are never any 'aliases'.  Since
          * the cache is physically tagged, the data can just be left in there.
          */
-        if (n_aliases == 0)
+        if (cpu_data->dcache.n_aliases == 0)
                 return;
 
-        all_aliases_mask = (1 << n_aliases) - 1;
-
         /*
          * Don't bother with the lookup and alias check if we have a
          * wide range to cover, just blow away the dcache in its
          * entirety instead. -- PFM.
          */
-        if (((end - start) >> PAGE_SHIFT) >= 64) {
+        if (((end - start) >> PAGE_SHIFT) >= MAX_DCACHE_PAGES)
                 flush_dcache_all();
-
-                if (vma->vm_flags & VM_EXEC)
-                        flush_icache_all();
-
-                return;
-        }
-
-        dir = pgd_offset(vma->vm_mm, p);
-        pud = pud_offset(dir, p);
-        pmd = pmd_offset(pud, p);
-        end = PAGE_ALIGN(end);
-
-        do {
-                if (pmd_none(*pmd) || pmd_bad(*pmd)) {
-                        p &= ~((1 << PMD_SHIFT) - 1);
-                        p += (1 << PMD_SHIFT);
-                        pmd++;
-
-                        continue;
-                }
-
-                pte = pte_offset_kernel(pmd, p);
-
-                do {
-                        entry = *pte;
-
-                        if ((pte_val(entry) & _PAGE_PRESENT)) {
-                                phys = pte_val(entry) & PTE_PHYS_MASK;
-
-                                if ((p ^ phys) & alias_mask) {
-                                        d |= 1 << ((p & alias_mask) >> PAGE_SHIFT);
-                                        d |= 1 << ((phys & alias_mask) >> PAGE_SHIFT);
-
-                                        if (d == all_aliases_mask)
-                                                goto loop_exit;
-                                }
-                        }
-
-                        pte++;
-                        p += PAGE_SIZE;
-                } while (p < end && ((unsigned long)pte & ~PAGE_MASK));
-                pmd++;
-        } while (p < end);
-
-loop_exit:
-        for (i = 0, select_bit = 0x1, addr_offset = 0x0; i < n_aliases;
-             i++, select_bit <<= 1, addr_offset += PAGE_SIZE)
-                if (d & select_bit) {
-                        (*__flush_dcache_segment_fn)(addr_offset, PAGE_SIZE);
-                        wmb();
-                }
+        else
+                __flush_cache_mm(vma->vm_mm, start, end);
 
         if (vma->vm_flags & VM_EXEC) {
                 /*
@@ -731,4 +772,3 @@ static void __flush_dcache_segment_4way(unsigned long start,
                 a3 += linesz;
         } while (a0 < a0e);
 }
-