aboutsummaryrefslogtreecommitdiffstats
path: root/mm
diff options
context:
space:
mode:
authorHugh Dickins <hugh@veritas.com>2005-10-29 21:16:21 -0400
committerLinus Torvalds <torvalds@g5.osdl.org>2005-10-30 00:40:40 -0400
commit872fec16d9a0ed3b75b8893aa217e49cca575ee5 (patch)
tree1dfc8b9f2754bdfff645188e497865c00201d535 /mm
parent46dea3d092d23a58b42499cc8a21de0fad079f4a (diff)
[PATCH] mm: init_mm without ptlock
First step in pushing down the page_table_lock. init_mm.page_table_lock has been used throughout the architectures (usually for ioremap): not to serialize kernel address space allocation (that's usually vmlist_lock), but because pud_alloc,pmd_alloc,pte_alloc_kernel expect caller holds it. Reverse that: don't lock or unlock init_mm.page_table_lock in any of the architectures; instead rely on pud_alloc,pmd_alloc,pte_alloc_kernel to take and drop it when allocating a new one, to check lest a racing task already did. Similarly no page_table_lock in vmalloc's map_vm_area. Some temporary ugliness in __pud_alloc and __pmd_alloc: since they also handle user mms, which are converted only by a later patch, for now they have to lock differently according to whether or not it's init_mm. If sources get muddled, there's a danger that an arch source taking init_mm.page_table_lock will be mixed with common source also taking it (or neither take it). So break the rules and make another change, which should break the build for such a mismatch: remove the redundant mm arg from pte_alloc_kernel (ppc64 scrapped its distinct ioremap_mm in 2.6.13). Exceptions: arm26 used pte_alloc_kernel on user mm, now pte_alloc_map; ia64 used pte_alloc_map on init_mm, now pte_alloc_kernel; parisc had bad args to pmd_alloc and pte_alloc_kernel in unused USE_HPPA_IOREMAP code; ppc64 map_io_page forgot to unlock on failure; ppc mmu_mapin_ram and ppc64 im_free took page_table_lock for no good reason. Signed-off-by: Hugh Dickins <hugh@veritas.com> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Diffstat (limited to 'mm')
-rw-r--r--mm/memory.c60
-rw-r--r--mm/vmalloc.c4
2 files changed, 28 insertions, 36 deletions
diff --git a/mm/memory.c b/mm/memory.c
index 692ad810263d..95a4553c75f7 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -307,28 +307,22 @@ out:
307 return pte_offset_map(pmd, address); 307 return pte_offset_map(pmd, address);
308} 308}
309 309
310pte_t fastcall * pte_alloc_kernel(struct mm_struct *mm, pmd_t *pmd, unsigned long address) 310pte_t fastcall * pte_alloc_kernel(pmd_t *pmd, unsigned long address)
311{ 311{
312 if (!pmd_present(*pmd)) { 312 if (!pmd_present(*pmd)) {
313 pte_t *new; 313 pte_t *new;
314 314
315 spin_unlock(&mm->page_table_lock); 315 new = pte_alloc_one_kernel(&init_mm, address);
316 new = pte_alloc_one_kernel(mm, address);
317 spin_lock(&mm->page_table_lock);
318 if (!new) 316 if (!new)
319 return NULL; 317 return NULL;
320 318
321 /* 319 spin_lock(&init_mm.page_table_lock);
322 * Because we dropped the lock, we should re-check the 320 if (pmd_present(*pmd))
323 * entry, as somebody else could have populated it..
324 */
325 if (pmd_present(*pmd)) {
326 pte_free_kernel(new); 321 pte_free_kernel(new);
327 goto out; 322 else
328 } 323 pmd_populate_kernel(&init_mm, pmd, new);
329 pmd_populate_kernel(mm, pmd, new); 324 spin_unlock(&init_mm.page_table_lock);
330 } 325 }
331out:
332 return pte_offset_kernel(pmd, address); 326 return pte_offset_kernel(pmd, address);
333} 327}
334 328
@@ -2097,30 +2091,30 @@ int __handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
2097#ifndef __PAGETABLE_PUD_FOLDED 2091#ifndef __PAGETABLE_PUD_FOLDED
2098/* 2092/*
2099 * Allocate page upper directory. 2093 * Allocate page upper directory.
2100 * 2094 * We've already handled the fast-path in-line.
2101 * We've already handled the fast-path in-line, and we own the
2102 * page table lock.
2103 */ 2095 */
2104pud_t fastcall *__pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address) 2096pud_t fastcall *__pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
2105{ 2097{
2106 pud_t *new; 2098 pud_t *new;
2107 2099
2108 spin_unlock(&mm->page_table_lock); 2100 if (mm != &init_mm) /* Temporary bridging hack */
2101 spin_unlock(&mm->page_table_lock);
2109 new = pud_alloc_one(mm, address); 2102 new = pud_alloc_one(mm, address);
2110 spin_lock(&mm->page_table_lock); 2103 if (!new) {
2111 if (!new) 2104 if (mm != &init_mm) /* Temporary bridging hack */
2105 spin_lock(&mm->page_table_lock);
2112 return NULL; 2106 return NULL;
2107 }
2113 2108
2114 /* 2109 spin_lock(&mm->page_table_lock);
2115 * Because we dropped the lock, we should re-check the
2116 * entry, as somebody else could have populated it..
2117 */
2118 if (pgd_present(*pgd)) { 2110 if (pgd_present(*pgd)) {
2119 pud_free(new); 2111 pud_free(new);
2120 goto out; 2112 goto out;
2121 } 2113 }
2122 pgd_populate(mm, pgd, new); 2114 pgd_populate(mm, pgd, new);
2123 out: 2115 out:
2116 if (mm == &init_mm) /* Temporary bridging hack */
2117 spin_unlock(&mm->page_table_lock);
2124 return pud_offset(pgd, address); 2118 return pud_offset(pgd, address);
2125} 2119}
2126#endif /* __PAGETABLE_PUD_FOLDED */ 2120#endif /* __PAGETABLE_PUD_FOLDED */
@@ -2128,24 +2122,22 @@ pud_t fastcall *__pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long addr
2128#ifndef __PAGETABLE_PMD_FOLDED 2122#ifndef __PAGETABLE_PMD_FOLDED
2129/* 2123/*
2130 * Allocate page middle directory. 2124 * Allocate page middle directory.
2131 * 2125 * We've already handled the fast-path in-line.
2132 * We've already handled the fast-path in-line, and we own the
2133 * page table lock.
2134 */ 2126 */
2135pmd_t fastcall *__pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) 2127pmd_t fastcall *__pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
2136{ 2128{
2137 pmd_t *new; 2129 pmd_t *new;
2138 2130
2139 spin_unlock(&mm->page_table_lock); 2131 if (mm != &init_mm) /* Temporary bridging hack */
2132 spin_unlock(&mm->page_table_lock);
2140 new = pmd_alloc_one(mm, address); 2133 new = pmd_alloc_one(mm, address);
2141 spin_lock(&mm->page_table_lock); 2134 if (!new) {
2142 if (!new) 2135 if (mm != &init_mm) /* Temporary bridging hack */
2136 spin_lock(&mm->page_table_lock);
2143 return NULL; 2137 return NULL;
2138 }
2144 2139
2145 /* 2140 spin_lock(&mm->page_table_lock);
2146 * Because we dropped the lock, we should re-check the
2147 * entry, as somebody else could have populated it..
2148 */
2149#ifndef __ARCH_HAS_4LEVEL_HACK 2141#ifndef __ARCH_HAS_4LEVEL_HACK
2150 if (pud_present(*pud)) { 2142 if (pud_present(*pud)) {
2151 pmd_free(new); 2143 pmd_free(new);
@@ -2161,6 +2153,8 @@ pmd_t fastcall *__pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long addr
2161#endif /* __ARCH_HAS_4LEVEL_HACK */ 2153#endif /* __ARCH_HAS_4LEVEL_HACK */
2162 2154
2163 out: 2155 out:
2156 if (mm == &init_mm) /* Temporary bridging hack */
2157 spin_unlock(&mm->page_table_lock);
2164 return pmd_offset(pud, address); 2158 return pmd_offset(pud, address);
2165} 2159}
2166#endif /* __PAGETABLE_PMD_FOLDED */ 2160#endif /* __PAGETABLE_PMD_FOLDED */
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 5e9120598799..54a90e83cb31 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -89,7 +89,7 @@ static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
89{ 89{
90 pte_t *pte; 90 pte_t *pte;
91 91
92 pte = pte_alloc_kernel(&init_mm, pmd, addr); 92 pte = pte_alloc_kernel(pmd, addr);
93 if (!pte) 93 if (!pte)
94 return -ENOMEM; 94 return -ENOMEM;
95 do { 95 do {
@@ -147,14 +147,12 @@ int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page ***pages)
147 147
148 BUG_ON(addr >= end); 148 BUG_ON(addr >= end);
149 pgd = pgd_offset_k(addr); 149 pgd = pgd_offset_k(addr);
150 spin_lock(&init_mm.page_table_lock);
151 do { 150 do {
152 next = pgd_addr_end(addr, end); 151 next = pgd_addr_end(addr, end);
153 err = vmap_pud_range(pgd, addr, next, prot, pages); 152 err = vmap_pud_range(pgd, addr, next, prot, pages);
154 if (err) 153 if (err)
155 break; 154 break;
156 } while (pgd++, addr = next, addr != end); 155 } while (pgd++, addr = next, addr != end);
157 spin_unlock(&init_mm.page_table_lock);
158 flush_cache_vmap((unsigned long) area->addr, end); 156 flush_cache_vmap((unsigned long) area->addr, end);
159 return err; 157 return err;
160} 158}