From a741e67969577163a4cfc78d7fd2753219087ef1 Mon Sep 17 00:00:00 2001 From: Benjamin Herrenschmidt Date: Tue, 10 Apr 2007 17:09:37 +1000 Subject: [POWERPC] Make tlb flush batch use lazy MMU mode The current tlb flush code on powerpc 64 bits has a subtle race since we lost the page table lock due to the possible faulting in of new PTEs after a previous one has been removed but before the corresponding hash entry has been evicted, which can leads to all sort of fatal problems. This patch reworks the batch code completely. It doesn't use the mmu_gather stuff anymore. Instead, we use the lazy mmu hooks that were added by the paravirt code. They have the nice property that the enter/leave lazy mmu mode pair is always fully contained by the PTE lock for a given range of PTEs. Thus we can guarantee that all batches are flushed on a given CPU before it drops that lock. We also generalize batching for any PTE update that require a flush. Batching is now enabled on a CPU by arch_enter_lazy_mmu_mode() and disabled by arch_leave_lazy_mmu_mode(). The code epects that this is always contained within a PTE lock section so no preemption can happen and no PTE insertion in that range from another CPU. When batching is enabled on a CPU, every PTE updates that need a hash flush will use the batch for that flush. Signed-off-by: Benjamin Herrenschmidt Signed-off-by: Paul Mackerras --- include/asm-powerpc/pgtable.h | 50 ++++++++++++++----------------------------- 1 file changed, 16 insertions(+), 34 deletions(-) (limited to 'include/asm-powerpc/pgtable.h') diff --git a/include/asm-powerpc/pgtable.h b/include/asm-powerpc/pgtable.h index 10f52743f4ff..c7142c7e0e05 100644 --- a/include/asm-powerpc/pgtable.h +++ b/include/asm-powerpc/pgtable.h @@ -272,7 +272,10 @@ static inline pte_t pte_mkhuge(pte_t pte) { return pte; } /* Atomic PTE updates */ -static inline unsigned long pte_update(pte_t *p, unsigned long clr) +static inline unsigned long pte_update(struct mm_struct *mm, + unsigned long addr, + pte_t *ptep, unsigned long clr, + int huge) { unsigned long old, tmp; @@ -283,20 +286,15 @@ static inline unsigned long pte_update(pte_t *p, unsigned long clr) andc %1,%0,%4 \n\ stdcx. %1,0,%3 \n\ bne- 1b" - : "=&r" (old), "=&r" (tmp), "=m" (*p) - : "r" (p), "r" (clr), "m" (*p), "i" (_PAGE_BUSY) + : "=&r" (old), "=&r" (tmp), "=m" (*ptep) + : "r" (ptep), "r" (clr), "m" (*ptep), "i" (_PAGE_BUSY) : "cc" ); + + if (old & _PAGE_HASHPTE) + hpte_need_flush(mm, addr, ptep, old, huge); return old; } -/* PTE updating functions, this function puts the PTE in the - * batch, doesn't actually triggers the hash flush immediately, - * you need to call flush_tlb_pending() to do that. - * Pass -1 for "normal" size (4K or 64K) - */ -extern void hpte_update(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, unsigned long pte, int huge); - static inline int __ptep_test_and_clear_young(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { @@ -304,11 +302,7 @@ static inline int __ptep_test_and_clear_young(struct mm_struct *mm, if ((pte_val(*ptep) & (_PAGE_ACCESSED | _PAGE_HASHPTE)) == 0) return 0; - old = pte_update(ptep, _PAGE_ACCESSED); - if (old & _PAGE_HASHPTE) { - hpte_update(mm, addr, ptep, old, 0); - flush_tlb_pending(); - } + old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0); return (old & _PAGE_ACCESSED) != 0; } #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG @@ -331,9 +325,7 @@ static inline int __ptep_test_and_clear_dirty(struct mm_struct *mm, if ((pte_val(*ptep) & _PAGE_DIRTY) == 0) return 0; - old = pte_update(ptep, _PAGE_DIRTY); - if (old & _PAGE_HASHPTE) - hpte_update(mm, addr, ptep, old, 0); + old = pte_update(mm, addr, ptep, _PAGE_DIRTY, 0); return (old & _PAGE_DIRTY) != 0; } #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_DIRTY @@ -352,9 +344,7 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, if ((pte_val(*ptep) & _PAGE_RW) == 0) return; - old = pte_update(ptep, _PAGE_RW); - if (old & _PAGE_HASHPTE) - hpte_update(mm, addr, ptep, old, 0); + old = pte_update(mm, addr, ptep, _PAGE_RW, 0); } /* @@ -378,7 +368,6 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, ({ \ int __dirty = __ptep_test_and_clear_dirty((__vma)->vm_mm, __address, \ __ptep); \ - flush_tlb_page(__vma, __address); \ __dirty; \ }) @@ -386,20 +375,14 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - unsigned long old = pte_update(ptep, ~0UL); - - if (old & _PAGE_HASHPTE) - hpte_update(mm, addr, ptep, old, 0); + unsigned long old = pte_update(mm, addr, ptep, ~0UL, 0); return __pte(old); } static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t * ptep) { - unsigned long old = pte_update(ptep, ~0UL); - - if (old & _PAGE_HASHPTE) - hpte_update(mm, addr, ptep, old, 0); + pte_update(mm, addr, ptep, ~0UL, 0); } /* @@ -408,10 +391,8 @@ static inline void pte_clear(struct mm_struct *mm, unsigned long addr, static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) { - if (pte_present(*ptep)) { + if (pte_present(*ptep)) pte_clear(mm, addr, ptep); - flush_tlb_pending(); - } pte = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS); *ptep = pte; } @@ -522,6 +503,7 @@ void pgtable_cache_init(void); return pt; } + #include #endif /* __ASSEMBLY__ */ -- cgit v1.2.2 From 621023072524fc0155ed16490255e1ea3aa11585 Mon Sep 17 00:00:00 2001 From: David Gibson Date: Tue, 24 Apr 2007 13:09:12 +1000 Subject: [POWERPC] Cleanup and fix breakage in tlbflush.h BenH's commit a741e67969577163a4cfc78d7fd2753219087ef1 in powerpc.git, although (AFAICT) only intended to affect ppc64, also has side-effects which break 44x. I think 40x, 8xx and Freescale Book E are also affected, though I haven't tested them. The problem lies in unconditionally removing flush_tlb_pending() from the versions of flush_tlb_mm(), flush_tlb_range() and flush_tlb_kernel_range() used on ppc64 - which are also used the embedded platforms mentioned above. The patch below cleans up the convoluted #ifdef logic in tlbflush.h, in the process restoring the necessary flushes for the software TLB platforms. There are three sets of definitions for the flushing hooks: the software TLB versions (revised to avoid using names which appear to related to TLB batching), the 32-bit hash based versions (external functions) amd the 64-bit hash based versions (which implement batching). It also moves the declaration of update_mmu_cache() to always be in tlbflush.h (previously it was in tlbflush.h except for PPC64, where it was in pgtable.h). Booted on Ebony (440GP) and compiled for 64-bit and 32-bit multiplatform. Signed-off-by: David Gibson Acked-by: Benjamin Herrenschmidt Signed-off-by: Paul Mackerras --- include/asm-powerpc/pgtable.h | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'include/asm-powerpc/pgtable.h') diff --git a/include/asm-powerpc/pgtable.h b/include/asm-powerpc/pgtable.h index c7142c7e0e05..19edb6982b81 100644 --- a/include/asm-powerpc/pgtable.h +++ b/include/asm-powerpc/pgtable.h @@ -448,16 +448,6 @@ extern pgd_t swapper_pg_dir[]; extern void paging_init(void); -/* - * This gets called at the end of handling a page fault, when - * the kernel has put a new PTE into the page table for the process. - * We use it to put a corresponding HPTE into the hash table - * ahead of time, instead of waiting for the inevitable extra - * hash-table miss exception. - */ -struct vm_area_struct; -extern void update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t); - /* Encode and de-code a swap entry */ #define __swp_type(entry) (((entry).val >> 1) & 0x3f) #define __swp_offset(entry) ((entry).val >> 8) -- cgit v1.2.2