aboutsummaryrefslogtreecommitdiffstats
path: root/mm/memory.c
diff options
context:
space:
mode:
authorHugh Dickins <hugh@veritas.com>2005-10-29 21:16:29 -0400
committerLinus Torvalds <torvalds@g5.osdl.org>2005-10-30 00:40:40 -0400
commit8f4f8c164cb4af1432cc25eda82928ea4519ba72 (patch)
tree49cd3c62069df1f8d6c863b9806923de16c10e8b /mm/memory.c
parent663b97f7efd001b0c56bd5fce059c5272725b86f (diff)
[PATCH] mm: unlink vma before pagetables
In most places the descent from pgd to pud to pmd to pte holds mmap_sem (exclusively or not), which ensures that free_pgtables cannot be freeing page tables from any level at the same time. But truncation and reverse mapping descend without mmap_sem. No problem: just make sure that a vma is unlinked from its prio_tree (or nonlinear list) and from its anon_vma list, after zapping the vma, but before freeing its page tables. Then neither vmtruncate nor rmap can reach that vma whose page tables are now volatile (nor do they need to reach it, since all its page entries have been zapped by this stage). The i_mmap_lock and anon_vma->lock already serialize this correctly; but the locking hierarchy is such that we cannot take them while holding page_table_lock. Well, we're trying to push that down anyway. So in this patch, move anon_vma_unlink and unlink_file_vma into free_pgtables, at the same time as moving page_table_lock around calls to unmap_vmas. tlb_gather_mmu and tlb_finish_mmu then fall outside the page_table_lock, but we made them preempt_disable and preempt_enable earlier; and a long source audit of all the architectures has shown no problem with removing page_table_lock from them. free_pgtables doesn't need page_table_lock for itself, nor for what it calls; tlb->mm->nr_ptes is usually protected by page_table_lock, but partly by non-exclusive mmap_sem - here it's decremented with exclusive mmap_sem, or mm_users 0. update_hiwater_rss and vm_unacct_memory don't need page_table_lock either. Signed-off-by: Hugh Dickins <hugh@veritas.com> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Diffstat (limited to 'mm/memory.c')
-rw-r--r--mm/memory.c12
1 files changed, 10 insertions, 2 deletions
diff --git a/mm/memory.c b/mm/memory.c
index 24ba688876d6..4ea89a2e3a83 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -260,6 +260,12 @@ void free_pgtables(struct mmu_gather **tlb, struct vm_area_struct *vma,
260 struct vm_area_struct *next = vma->vm_next; 260 struct vm_area_struct *next = vma->vm_next;
261 unsigned long addr = vma->vm_start; 261 unsigned long addr = vma->vm_start;
262 262
263 /*
264 * Hide vma from rmap and vmtruncate before freeing pgtables
265 */
266 anon_vma_unlink(vma);
267 unlink_file_vma(vma);
268
263 if (is_hugepage_only_range(vma->vm_mm, addr, HPAGE_SIZE)) { 269 if (is_hugepage_only_range(vma->vm_mm, addr, HPAGE_SIZE)) {
264 hugetlb_free_pgd_range(tlb, addr, vma->vm_end, 270 hugetlb_free_pgd_range(tlb, addr, vma->vm_end,
265 floor, next? next->vm_start: ceiling); 271 floor, next? next->vm_start: ceiling);
@@ -272,6 +278,8 @@ void free_pgtables(struct mmu_gather **tlb, struct vm_area_struct *vma,
272 HPAGE_SIZE)) { 278 HPAGE_SIZE)) {
273 vma = next; 279 vma = next;
274 next = vma->vm_next; 280 next = vma->vm_next;
281 anon_vma_unlink(vma);
282 unlink_file_vma(vma);
275 } 283 }
276 free_pgd_range(tlb, addr, vma->vm_end, 284 free_pgd_range(tlb, addr, vma->vm_end,
277 floor, next? next->vm_start: ceiling); 285 floor, next? next->vm_start: ceiling);
@@ -798,12 +806,12 @@ unsigned long zap_page_range(struct vm_area_struct *vma, unsigned long address,
798 } 806 }
799 807
800 lru_add_drain(); 808 lru_add_drain();
801 spin_lock(&mm->page_table_lock);
802 tlb = tlb_gather_mmu(mm, 0); 809 tlb = tlb_gather_mmu(mm, 0);
803 update_hiwater_rss(mm); 810 update_hiwater_rss(mm);
811 spin_lock(&mm->page_table_lock);
804 end = unmap_vmas(&tlb, mm, vma, address, end, &nr_accounted, details); 812 end = unmap_vmas(&tlb, mm, vma, address, end, &nr_accounted, details);
805 tlb_finish_mmu(tlb, address, end);
806 spin_unlock(&mm->page_table_lock); 813 spin_unlock(&mm->page_table_lock);
814 tlb_finish_mmu(tlb, address, end);
807 return end; 815 return end;
808} 816}
809 817