diff options
author | Hugh Dickins <hugh@veritas.com> | 2005-10-29 21:16:23 -0400 |
---|---|---|
committer | Linus Torvalds <torvalds@g5.osdl.org> | 2005-10-30 00:40:40 -0400 |
commit | c74df32c724a1652ad8399b4891bb02c9d43743a (patch) | |
tree | 5a79d56fdcf7dc2053a277dbf6db7c3b339e9659 /mm/hugetlb.c | |
parent | 1bb3630e89cb8a7b3d3807629c20c5bad88290ff (diff) |
[PATCH] mm: ptd_alloc take ptlock
Second step in pushing down the page_table_lock. Remove the temporary
bridging hack from __pud_alloc, __pmd_alloc, __pte_alloc: expect callers not
to hold page_table_lock, whether it's on init_mm or a user mm; take
page_table_lock internally to check if a racing task already allocated.
Convert their callers from common code. But avoid coming back to change them
again later: instead of moving the spin_lock(&mm->page_table_lock) down,
switch over to new macros pte_alloc_map_lock and pte_unmap_unlock, which
encapsulate the mapping+locking and unlocking+unmapping together, and in the
end may use alternatives to the mm page_table_lock itself.
These callers all hold mmap_sem (some exclusively, some not), so at no level
can a page table be whipped away from beneath them; and pte_alloc uses the
"atomic" pmd_present to test whether it needs to allocate. It appears that on
all arches we can safely descend without page_table_lock.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Diffstat (limited to 'mm/hugetlb.c')
-rw-r--r-- | mm/hugetlb.c | 12 |
1 files changed, 8 insertions, 4 deletions
diff --git a/mm/hugetlb.c b/mm/hugetlb.c index ac5f044bf51..ea0826ff266 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c | |||
@@ -277,12 +277,15 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, | |||
277 | unsigned long addr; | 277 | unsigned long addr; |
278 | 278 | ||
279 | for (addr = vma->vm_start; addr < vma->vm_end; addr += HPAGE_SIZE) { | 279 | for (addr = vma->vm_start; addr < vma->vm_end; addr += HPAGE_SIZE) { |
280 | src_pte = huge_pte_offset(src, addr); | ||
281 | if (!src_pte) | ||
282 | continue; | ||
280 | dst_pte = huge_pte_alloc(dst, addr); | 283 | dst_pte = huge_pte_alloc(dst, addr); |
281 | if (!dst_pte) | 284 | if (!dst_pte) |
282 | goto nomem; | 285 | goto nomem; |
286 | spin_lock(&dst->page_table_lock); | ||
283 | spin_lock(&src->page_table_lock); | 287 | spin_lock(&src->page_table_lock); |
284 | src_pte = huge_pte_offset(src, addr); | 288 | if (!pte_none(*src_pte)) { |
285 | if (src_pte && !pte_none(*src_pte)) { | ||
286 | entry = *src_pte; | 289 | entry = *src_pte; |
287 | ptepage = pte_page(entry); | 290 | ptepage = pte_page(entry); |
288 | get_page(ptepage); | 291 | get_page(ptepage); |
@@ -290,6 +293,7 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, | |||
290 | set_huge_pte_at(dst, addr, dst_pte, entry); | 293 | set_huge_pte_at(dst, addr, dst_pte, entry); |
291 | } | 294 | } |
292 | spin_unlock(&src->page_table_lock); | 295 | spin_unlock(&src->page_table_lock); |
296 | spin_unlock(&dst->page_table_lock); | ||
293 | } | 297 | } |
294 | return 0; | 298 | return 0; |
295 | 299 | ||
@@ -354,7 +358,6 @@ int hugetlb_prefault(struct address_space *mapping, struct vm_area_struct *vma) | |||
354 | 358 | ||
355 | hugetlb_prefault_arch_hook(mm); | 359 | hugetlb_prefault_arch_hook(mm); |
356 | 360 | ||
357 | spin_lock(&mm->page_table_lock); | ||
358 | for (addr = vma->vm_start; addr < vma->vm_end; addr += HPAGE_SIZE) { | 361 | for (addr = vma->vm_start; addr < vma->vm_end; addr += HPAGE_SIZE) { |
359 | unsigned long idx; | 362 | unsigned long idx; |
360 | pte_t *pte = huge_pte_alloc(mm, addr); | 363 | pte_t *pte = huge_pte_alloc(mm, addr); |
@@ -389,11 +392,12 @@ int hugetlb_prefault(struct address_space *mapping, struct vm_area_struct *vma) | |||
389 | goto out; | 392 | goto out; |
390 | } | 393 | } |
391 | } | 394 | } |
395 | spin_lock(&mm->page_table_lock); | ||
392 | add_mm_counter(mm, file_rss, HPAGE_SIZE / PAGE_SIZE); | 396 | add_mm_counter(mm, file_rss, HPAGE_SIZE / PAGE_SIZE); |
393 | set_huge_pte_at(mm, addr, pte, make_huge_pte(vma, page)); | 397 | set_huge_pte_at(mm, addr, pte, make_huge_pte(vma, page)); |
398 | spin_unlock(&mm->page_table_lock); | ||
394 | } | 399 | } |
395 | out: | 400 | out: |
396 | spin_unlock(&mm->page_table_lock); | ||
397 | return ret; | 401 | return ret; |
398 | } | 402 | } |
399 | 403 | ||