[POWERPC] Make tlb flush batch use lazy MMU mode

The current tlb flush code on powerpc 64 bits has a subtle race since we
lost the page table lock due to the possible faulting in of new PTEs
after a previous one has been removed but before the corresponding hash
entry has been evicted, which can leads to all sort of fatal problems.

This patch reworks the batch code completely. It doesn't use the mmu_gather
stuff anymore. Instead, we use the lazy mmu hooks that were added by the
paravirt code. They have the nice property that the enter/leave lazy mmu
mode pair is always fully contained by the PTE lock for a given range
of PTEs. Thus we can guarantee that all batches are flushed on a given
CPU before it drops that lock.

We also generalize batching for any PTE update that require a flush.

Batching is now enabled on a CPU by arch_enter_lazy_mmu_mode() and
disabled by arch_leave_lazy_mmu_mode(). The code epects that this is
always contained within a PTE lock section so no preemption can happen
and no PTE insertion in that range from another CPU. When batching
is enabled on a CPU, every PTE updates that need a hash flush will
use the batch for that flush.

Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: Paul Mackerras <paulus@samba.org>

1 files changed, 16 insertions, 34 deletions

diff --git a/include/asm-powerpc/pgtable.h b/include/asm-powerpc/pgtable.h
index 10f52743f4f..c7142c7e0e0 100644
--- a/include/asm-powerpc/pgtable.h
+++ b/include/asm-powerpc/pgtable.h
@@ -272,7 +272,10 @@ static inline pte_t pte_mkhuge(pte_t pte) {
         return pte; }
 
 /* Atomic PTE updates */
-static inline unsigned long pte_update(pte_t *p, unsigned long clr)
+static inline unsigned long pte_update(struct mm_struct *mm,
+                                       unsigned long addr,
+                                       pte_t *ptep, unsigned long clr,
+                                       int huge)
 {
         unsigned long old, tmp;
 
@@ -283,20 +286,15 @@ static inline unsigned long pte_update(pte_t *p, unsigned long clr)
         andc    %1,%0,%4 \n\
         stdcx.  %1,0,%3 \n\
         bne-    1b"
-        : "=&r" (old), "=&r" (tmp), "=m" (*p)
-        : "r" (p), "r" (clr), "m" (*p), "i" (_PAGE_BUSY)
+        : "=&r" (old), "=&r" (tmp), "=m" (*ptep)
+        : "r" (ptep), "r" (clr), "m" (*ptep), "i" (_PAGE_BUSY)
         : "cc" );
+
+        if (old & _PAGE_HASHPTE)
+                hpte_need_flush(mm, addr, ptep, old, huge);
         return old;
 }
 
-/* PTE updating functions, this function puts the PTE in the
- * batch, doesn't actually triggers the hash flush immediately,
- * you need to call flush_tlb_pending() to do that.
- * Pass -1 for "normal" size (4K or 64K)
- */
-extern void hpte_update(struct mm_struct *mm, unsigned long addr,
-                        pte_t *ptep, unsigned long pte, int huge);
-
 static inline int __ptep_test_and_clear_young(struct mm_struct *mm,
                                               unsigned long addr, pte_t *ptep)
 {
@@ -304,11 +302,7 @@ static inline int __ptep_test_and_clear_young(struct mm_struct *mm,
 
         if ((pte_val(*ptep) & (_PAGE_ACCESSED | _PAGE_HASHPTE)) == 0)
                 return 0;
-        old = pte_update(ptep, _PAGE_ACCESSED);
-        if (old & _PAGE_HASHPTE) {
-                hpte_update(mm, addr, ptep, old, 0);
-                flush_tlb_pending();
-        }
+        old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0);
         return (old & _PAGE_ACCESSED) != 0;
 }
 #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
@@ -331,9 +325,7 @@ static inline int __ptep_test_and_clear_dirty(struct mm_struct *mm,
 
         if ((pte_val(*ptep) & _PAGE_DIRTY) == 0)
                 return 0;
-        old = pte_update(ptep, _PAGE_DIRTY);
-        if (old & _PAGE_HASHPTE)
-                hpte_update(mm, addr, ptep, old, 0);
+        old = pte_update(mm, addr, ptep, _PAGE_DIRTY, 0);
         return (old & _PAGE_DIRTY) != 0;
 }
 #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_DIRTY
@@ -352,9 +344,7 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr,
 
         if ((pte_val(*ptep) & _PAGE_RW) == 0)
                 return;
-        old = pte_update(ptep, _PAGE_RW);
-        if (old & _PAGE_HASHPTE)
-                hpte_update(mm, addr, ptep, old, 0);
+        old = pte_update(mm, addr, ptep, _PAGE_RW, 0);
 }
 
 /*
@@ -378,7 +368,6 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr,
 ({                                                                      \
         int __dirty = __ptep_test_and_clear_dirty((__vma)->vm_mm, __address, \
                                                   __ptep);              \
-        flush_tlb_page(__vma, __address);                               \
         __dirty;                                                        \
 })
 
@@ -386,20 +375,14 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr,
 static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
                                        unsigned long addr, pte_t *ptep)
 {
-        unsigned long old = pte_update(ptep, ~0UL);
-
-        if (old & _PAGE_HASHPTE)
-                hpte_update(mm, addr, ptep, old, 0);
+        unsigned long old = pte_update(mm, addr, ptep, ~0UL, 0);
         return __pte(old);
 }
 
 static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
                              pte_t * ptep)
 {
-        unsigned long old = pte_update(ptep, ~0UL);
-
-        if (old & _PAGE_HASHPTE)
-                hpte_update(mm, addr, ptep, old, 0);
+        pte_update(mm, addr, ptep, ~0UL, 0);
 }
 
 /*
@@ -408,10 +391,8 @@ static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
 static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
                               pte_t *ptep, pte_t pte)
 {
-        if (pte_present(*ptep)) {
+        if (pte_present(*ptep))
                 pte_clear(mm, addr, ptep);
-                flush_tlb_pending();
-        }
         pte = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS);
         *ptep = pte;
 }
@@ -522,6 +503,7 @@ void pgtable_cache_init(void);
         return pt;
 }
 
+
 #include <asm-generic/pgtable.h>
 
 #endif /* __ASSEMBLY__ */