aboutsummaryrefslogtreecommitdiffstats
path: root/include/asm-powerpc/tlbflush.h
diff options
context:
space:
mode:
authorBenjamin Herrenschmidt <benh@kernel.crashing.org>2007-04-10 03:09:37 -0400
committerPaul Mackerras <paulus@samba.org>2007-04-12 14:09:38 -0400
commita741e67969577163a4cfc78d7fd2753219087ef1 (patch)
treebac4162aaf15367e896429afa60465e201c9204c /include/asm-powerpc/tlbflush.h
parente4ee3891db35aa9a069bb403c2a66a8fbfa274d6 (diff)
[POWERPC] Make tlb flush batch use lazy MMU mode
The current tlb flush code on powerpc 64 bits has a subtle race since we lost the page table lock due to the possible faulting in of new PTEs after a previous one has been removed but before the corresponding hash entry has been evicted, which can leads to all sort of fatal problems. This patch reworks the batch code completely. It doesn't use the mmu_gather stuff anymore. Instead, we use the lazy mmu hooks that were added by the paravirt code. They have the nice property that the enter/leave lazy mmu mode pair is always fully contained by the PTE lock for a given range of PTEs. Thus we can guarantee that all batches are flushed on a given CPU before it drops that lock. We also generalize batching for any PTE update that require a flush. Batching is now enabled on a CPU by arch_enter_lazy_mmu_mode() and disabled by arch_leave_lazy_mmu_mode(). The code epects that this is always contained within a PTE lock section so no preemption can happen and no PTE insertion in that range from another CPU. When batching is enabled on a CPU, every PTE updates that need a hash flush will use the batch for that flush. Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org> Signed-off-by: Paul Mackerras <paulus@samba.org>
Diffstat (limited to 'include/asm-powerpc/tlbflush.h')
-rw-r--r--include/asm-powerpc/tlbflush.h39
1 files changed, 25 insertions, 14 deletions
diff --git a/include/asm-powerpc/tlbflush.h b/include/asm-powerpc/tlbflush.h
index 93c7d0c7230f..0bc5a5e506be 100644
--- a/include/asm-powerpc/tlbflush.h
+++ b/include/asm-powerpc/tlbflush.h
@@ -28,25 +28,41 @@ struct mm_struct;
28#define PPC64_TLB_BATCH_NR 192 28#define PPC64_TLB_BATCH_NR 192
29 29
30struct ppc64_tlb_batch { 30struct ppc64_tlb_batch {
31 unsigned long index; 31 int active;
32 struct mm_struct *mm; 32 unsigned long index;
33 real_pte_t pte[PPC64_TLB_BATCH_NR]; 33 struct mm_struct *mm;
34 unsigned long vaddr[PPC64_TLB_BATCH_NR]; 34 real_pte_t pte[PPC64_TLB_BATCH_NR];
35 unsigned int psize; 35 unsigned long vaddr[PPC64_TLB_BATCH_NR];
36 unsigned int psize;
36}; 37};
37DECLARE_PER_CPU(struct ppc64_tlb_batch, ppc64_tlb_batch); 38DECLARE_PER_CPU(struct ppc64_tlb_batch, ppc64_tlb_batch);
38 39
39extern void __flush_tlb_pending(struct ppc64_tlb_batch *batch); 40extern void __flush_tlb_pending(struct ppc64_tlb_batch *batch);
40 41
41static inline void flush_tlb_pending(void) 42extern void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
43 pte_t *ptep, unsigned long pte, int huge);
44
45#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE
46
47static inline void arch_enter_lazy_mmu_mode(void)
48{
49 struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);
50
51 batch->active = 1;
52}
53
54static inline void arch_leave_lazy_mmu_mode(void)
42{ 55{
43 struct ppc64_tlb_batch *batch = &get_cpu_var(ppc64_tlb_batch); 56 struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);
44 57
45 if (batch->index) 58 if (batch->index)
46 __flush_tlb_pending(batch); 59 __flush_tlb_pending(batch);
47 put_cpu_var(ppc64_tlb_batch); 60 batch->active = 0;
48} 61}
49 62
63#define arch_flush_lazy_mmu_mode() do {} while (0)
64
65
50extern void flush_hash_page(unsigned long va, real_pte_t pte, int psize, 66extern void flush_hash_page(unsigned long va, real_pte_t pte, int psize,
51 int local); 67 int local);
52extern void flush_hash_range(unsigned long number, int local); 68extern void flush_hash_range(unsigned long number, int local);
@@ -88,15 +104,12 @@ extern void update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t);
88 104
89static inline void flush_tlb_mm(struct mm_struct *mm) 105static inline void flush_tlb_mm(struct mm_struct *mm)
90{ 106{
91 flush_tlb_pending();
92} 107}
93 108
94static inline void flush_tlb_page(struct vm_area_struct *vma, 109static inline void flush_tlb_page(struct vm_area_struct *vma,
95 unsigned long vmaddr) 110 unsigned long vmaddr)
96{ 111{
97#ifdef CONFIG_PPC64 112#ifndef CONFIG_PPC64
98 flush_tlb_pending();
99#else
100 _tlbie(vmaddr); 113 _tlbie(vmaddr);
101#endif 114#endif
102} 115}
@@ -112,13 +125,11 @@ static inline void flush_tlb_page_nohash(struct vm_area_struct *vma,
112static inline void flush_tlb_range(struct vm_area_struct *vma, 125static inline void flush_tlb_range(struct vm_area_struct *vma,
113 unsigned long start, unsigned long end) 126 unsigned long start, unsigned long end)
114{ 127{
115 flush_tlb_pending();
116} 128}
117 129
118static inline void flush_tlb_kernel_range(unsigned long start, 130static inline void flush_tlb_kernel_range(unsigned long start,
119 unsigned long end) 131 unsigned long end)
120{ 132{
121 flush_tlb_pending();
122} 133}
123 134
124#else /* 6xx, 7xx, 7xxx cpus */ 135#else /* 6xx, 7xx, 7xxx cpus */