3 files changed, 41 insertions, 49 deletions

diff --git a/include/asm-powerpc/pgtable.h b/include/asm-powerpc/pgtable.h
index 10f52743f4ff..c7142c7e0e05 100644
--- a/include/asm-powerpc/pgtable.h
+++ b/include/asm-powerpc/pgtable.h
@@ -272,7 +272,10 @@ static inline pte_t pte_mkhuge(pte_t pte) {
         return pte; }
 
 /* Atomic PTE updates */
-static inline unsigned long pte_update(pte_t *p, unsigned long clr)
+static inline unsigned long pte_update(struct mm_struct *mm,
+                                       unsigned long addr,
+                                       pte_t *ptep, unsigned long clr,
+                                       int huge)
 {
         unsigned long old, tmp;
 
@@ -283,20 +286,15 @@ static inline unsigned long pte_update(pte_t *p, unsigned long clr)
         andc    %1,%0,%4 \n\
         stdcx.  %1,0,%3 \n\
         bne-    1b"
-        : "=&r" (old), "=&r" (tmp), "=m" (*p)
-        : "r" (p), "r" (clr), "m" (*p), "i" (_PAGE_BUSY)
+        : "=&r" (old), "=&r" (tmp), "=m" (*ptep)
+        : "r" (ptep), "r" (clr), "m" (*ptep), "i" (_PAGE_BUSY)
         : "cc" );
+
+        if (old & _PAGE_HASHPTE)
+                hpte_need_flush(mm, addr, ptep, old, huge);
         return old;
 }
 
-/* PTE updating functions, this function puts the PTE in the
- * batch, doesn't actually triggers the hash flush immediately,
- * you need to call flush_tlb_pending() to do that.
- * Pass -1 for "normal" size (4K or 64K)
- */
-extern void hpte_update(struct mm_struct *mm, unsigned long addr,
-                        pte_t *ptep, unsigned long pte, int huge);
-
 static inline int __ptep_test_and_clear_young(struct mm_struct *mm,
                                               unsigned long addr, pte_t *ptep)
 {
@@ -304,11 +302,7 @@ static inline int __ptep_test_and_clear_young(struct mm_struct *mm,
 
         if ((pte_val(*ptep) & (_PAGE_ACCESSED | _PAGE_HASHPTE)) == 0)
                 return 0;
-        old = pte_update(ptep, _PAGE_ACCESSED);
-        if (old & _PAGE_HASHPTE) {
-                hpte_update(mm, addr, ptep, old, 0);
-                flush_tlb_pending();
-        }
+        old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0);
         return (old & _PAGE_ACCESSED) != 0;
 }
 #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
@@ -331,9 +325,7 @@ static inline int __ptep_test_and_clear_dirty(struct mm_struct *mm,
 
         if ((pte_val(*ptep) & _PAGE_DIRTY) == 0)
                 return 0;
-        old = pte_update(ptep, _PAGE_DIRTY);
-        if (old & _PAGE_HASHPTE)
-                hpte_update(mm, addr, ptep, old, 0);
+        old = pte_update(mm, addr, ptep, _PAGE_DIRTY, 0);
         return (old & _PAGE_DIRTY) != 0;
 }
 #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_DIRTY
@@ -352,9 +344,7 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr,
 
         if ((pte_val(*ptep) & _PAGE_RW) == 0)
                 return;
-        old = pte_update(ptep, _PAGE_RW);
-        if (old & _PAGE_HASHPTE)
-                hpte_update(mm, addr, ptep, old, 0);
+        old = pte_update(mm, addr, ptep, _PAGE_RW, 0);
 }
 
 /*
@@ -378,7 +368,6 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr,
 ({                                                                      \
         int __dirty = __ptep_test_and_clear_dirty((__vma)->vm_mm, __address, \
                                                   __ptep);              \
-        flush_tlb_page(__vma, __address);                               \
         __dirty;                                                        \
 })
 
@@ -386,20 +375,14 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr,
 static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
                                        unsigned long addr, pte_t *ptep)
 {
-        unsigned long old = pte_update(ptep, ~0UL);
-
-        if (old & _PAGE_HASHPTE)
-                hpte_update(mm, addr, ptep, old, 0);
+        unsigned long old = pte_update(mm, addr, ptep, ~0UL, 0);
         return __pte(old);
 }
 
 static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
                              pte_t * ptep)
 {
-        unsigned long old = pte_update(ptep, ~0UL);
-
-        if (old & _PAGE_HASHPTE)
-                hpte_update(mm, addr, ptep, old, 0);
+        pte_update(mm, addr, ptep, ~0UL, 0);
 }
 
 /*
@@ -408,10 +391,8 @@ static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
 static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
                               pte_t *ptep, pte_t pte)
 {
-        if (pte_present(*ptep)) {
+        if (pte_present(*ptep))
                 pte_clear(mm, addr, ptep);
-                flush_tlb_pending();
-        }
         pte = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS);
         *ptep = pte;
 }
@@ -522,6 +503,7 @@ void pgtable_cache_init(void);
         return pt;
 }
 
+
 #include <asm-generic/pgtable.h>
 
 #endif /* __ASSEMBLY__ */
diff --git a/include/asm-powerpc/tlb.h b/include/asm-powerpc/tlb.h
index 4e2a834683fb..0a17682663d8 100644
--- a/include/asm-powerpc/tlb.h
+++ b/include/asm-powerpc/tlb.h
@@ -38,7 +38,6 @@ extern void pte_free_finish(void);
 
 static inline void tlb_flush(struct mmu_gather *tlb)
 {
-        flush_tlb_pending();
         pte_free_finish();
 }
 
diff --git a/include/asm-powerpc/tlbflush.h b/include/asm-powerpc/tlbflush.h
index 93c7d0c7230f..0bc5a5e506be 100644
--- a/include/asm-powerpc/tlbflush.h
+++ b/include/asm-powerpc/tlbflush.h
@@ -28,25 +28,41 @@ struct mm_struct;
 #define PPC64_TLB_BATCH_NR 192
 
 struct ppc64_tlb_batch {
-        unsigned long index;
-        struct mm_struct *mm;
-        real_pte_t pte[PPC64_TLB_BATCH_NR];
-        unsigned long vaddr[PPC64_TLB_BATCH_NR];
-        unsigned int psize;
+        int                     active;
+        unsigned long           index;
+        struct mm_struct        *mm;
+        real_pte_t              pte[PPC64_TLB_BATCH_NR];
+        unsigned long           vaddr[PPC64_TLB_BATCH_NR];
+        unsigned int            psize;
 };
 DECLARE_PER_CPU(struct ppc64_tlb_batch, ppc64_tlb_batch);
 
 extern void __flush_tlb_pending(struct ppc64_tlb_batch *batch);
 
-static inline void flush_tlb_pending(void)
+extern void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
+                            pte_t *ptep, unsigned long pte, int huge);
+
+#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE
+
+static inline void arch_enter_lazy_mmu_mode(void)
+{
+        struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);
+
+        batch->active = 1;
+}
+
+static inline void arch_leave_lazy_mmu_mode(void)
 {
-        struct ppc64_tlb_batch *batch = &get_cpu_var(ppc64_tlb_batch);
+        struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);
 
         if (batch->index)
                 __flush_tlb_pending(batch);
-        put_cpu_var(ppc64_tlb_batch);
+        batch->active = 0;
 }
 
+#define arch_flush_lazy_mmu_mode()      do {} while (0)
+
+
 extern void flush_hash_page(unsigned long va, real_pte_t pte, int psize,
                             int local);
 extern void flush_hash_range(unsigned long number, int local);
@@ -88,15 +104,12 @@ extern void update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t);
 
 static inline void flush_tlb_mm(struct mm_struct *mm)
 {
-        flush_tlb_pending();
 }
 
 static inline void flush_tlb_page(struct vm_area_struct *vma,
                                 unsigned long vmaddr)
 {
-#ifdef CONFIG_PPC64
-        flush_tlb_pending();
-#else
+#ifndef CONFIG_PPC64
         _tlbie(vmaddr);
 #endif
 }
@@ -112,13 +125,11 @@ static inline void flush_tlb_page_nohash(struct vm_area_struct *vma,
 static inline void flush_tlb_range(struct vm_area_struct *vma,
                 unsigned long start, unsigned long end)
 {
-        flush_tlb_pending();
 }
 
 static inline void flush_tlb_kernel_range(unsigned long start,
                 unsigned long end)
 {
-        flush_tlb_pending();
 }
 
 #else   /* 6xx, 7xx, 7xxx cpus */