diff options
author | David Gibson <david@gibson.dropbear.id.au> | 2005-06-21 20:14:44 -0400 |
---|---|---|
committer | Linus Torvalds <torvalds@ppc970.osdl.org> | 2005-06-21 21:46:15 -0400 |
commit | 63551ae0feaaa23807ebea60de1901564bbef32e (patch) | |
tree | f6f97f60f83c3e9813bdfcc6039c499997b1ea10 /arch/sh/mm | |
parent | 1e7e5a9048b30c57ba1ddaa6cdf59b21b65cde99 (diff) |
[PATCH] Hugepage consolidation
A lot of the code in arch/*/mm/hugetlbpage.c is quite similar. This patch
attempts to consolidate a lot of the code across the arch's, putting the
combined version in mm/hugetlb.c. There are a couple of uglyish hacks in
order to covert all the hugepage archs, but the result is a very large
reduction in the total amount of code. It also means things like hugepage
lazy allocation could be implemented in one place, instead of six.
Tested, at least a little, on ppc64, i386 and x86_64.
Notes:
- this patch changes the meaning of set_huge_pte() to be more
analagous to set_pte()
- does SH4 need s special huge_ptep_get_and_clear()??
Acked-by: William Lee Irwin <wli@holomorphy.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Diffstat (limited to 'arch/sh/mm')
-rw-r--r-- | arch/sh/mm/hugetlbpage.c | 196 |
1 files changed, 24 insertions, 172 deletions
diff --git a/arch/sh/mm/hugetlbpage.c b/arch/sh/mm/hugetlbpage.c index 1f897bab2318..95bb1a6c6060 100644 --- a/arch/sh/mm/hugetlbpage.c +++ b/arch/sh/mm/hugetlbpage.c | |||
@@ -24,7 +24,7 @@ | |||
24 | #include <asm/tlbflush.h> | 24 | #include <asm/tlbflush.h> |
25 | #include <asm/cacheflush.h> | 25 | #include <asm/cacheflush.h> |
26 | 26 | ||
27 | static pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr) | 27 | pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr) |
28 | { | 28 | { |
29 | pgd_t *pgd; | 29 | pgd_t *pgd; |
30 | pmd_t *pmd; | 30 | pmd_t *pmd; |
@@ -39,7 +39,7 @@ static pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr) | |||
39 | return pte; | 39 | return pte; |
40 | } | 40 | } |
41 | 41 | ||
42 | static pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) | 42 | pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) |
43 | { | 43 | { |
44 | pgd_t *pgd; | 44 | pgd_t *pgd; |
45 | pmd_t *pmd; | 45 | pmd_t *pmd; |
@@ -56,28 +56,34 @@ static pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) | |||
56 | 56 | ||
57 | #define mk_pte_huge(entry) do { pte_val(entry) |= _PAGE_SZHUGE; } while (0) | 57 | #define mk_pte_huge(entry) do { pte_val(entry) |= _PAGE_SZHUGE; } while (0) |
58 | 58 | ||
59 | static void set_huge_pte(struct mm_struct *mm, struct vm_area_struct *vma, | 59 | void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, |
60 | struct page *page, pte_t * page_table, int write_access) | 60 | pte_t *ptep, pte_t entry) |
61 | { | 61 | { |
62 | unsigned long i; | 62 | int i; |
63 | pte_t entry; | ||
64 | 63 | ||
65 | add_mm_counter(mm, rss, HPAGE_SIZE / PAGE_SIZE); | 64 | for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) { |
65 | set_pte_at(mm, addr, ptep, entry); | ||
66 | ptep++; | ||
67 | addr += PAGE_SIZE; | ||
68 | pte_val(entry) += PAGE_SIZE; | ||
69 | } | ||
70 | } | ||
66 | 71 | ||
67 | if (write_access) | 72 | pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, |
68 | entry = pte_mkwrite(pte_mkdirty(mk_pte(page, | 73 | pte_t *ptep) |
69 | vma->vm_page_prot))); | 74 | { |
70 | else | 75 | pte_t entry; |
71 | entry = pte_wrprotect(mk_pte(page, vma->vm_page_prot)); | 76 | int i; |
72 | entry = pte_mkyoung(entry); | ||
73 | mk_pte_huge(entry); | ||
74 | 77 | ||
75 | for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) { | 78 | entry = *ptep; |
76 | set_pte(page_table, entry); | ||
77 | page_table++; | ||
78 | 79 | ||
79 | pte_val(entry) += PAGE_SIZE; | 80 | for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) { |
81 | pte_clear(mm, addr, ptep); | ||
82 | addr += PAGE_SIZE; | ||
83 | ptep++; | ||
80 | } | 84 | } |
85 | |||
86 | return entry; | ||
81 | } | 87 | } |
82 | 88 | ||
83 | /* | 89 | /* |
@@ -92,79 +98,6 @@ int is_aligned_hugepage_range(unsigned long addr, unsigned long len) | |||
92 | return 0; | 98 | return 0; |
93 | } | 99 | } |
94 | 100 | ||
95 | int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, | ||
96 | struct vm_area_struct *vma) | ||
97 | { | ||
98 | pte_t *src_pte, *dst_pte, entry; | ||
99 | struct page *ptepage; | ||
100 | unsigned long addr = vma->vm_start; | ||
101 | unsigned long end = vma->vm_end; | ||
102 | int i; | ||
103 | |||
104 | while (addr < end) { | ||
105 | dst_pte = huge_pte_alloc(dst, addr); | ||
106 | if (!dst_pte) | ||
107 | goto nomem; | ||
108 | src_pte = huge_pte_offset(src, addr); | ||
109 | BUG_ON(!src_pte || pte_none(*src_pte)); | ||
110 | entry = *src_pte; | ||
111 | ptepage = pte_page(entry); | ||
112 | get_page(ptepage); | ||
113 | for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) { | ||
114 | set_pte(dst_pte, entry); | ||
115 | pte_val(entry) += PAGE_SIZE; | ||
116 | dst_pte++; | ||
117 | } | ||
118 | add_mm_counter(dst, rss, HPAGE_SIZE / PAGE_SIZE); | ||
119 | addr += HPAGE_SIZE; | ||
120 | } | ||
121 | return 0; | ||
122 | |||
123 | nomem: | ||
124 | return -ENOMEM; | ||
125 | } | ||
126 | |||
127 | int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, | ||
128 | struct page **pages, struct vm_area_struct **vmas, | ||
129 | unsigned long *position, int *length, int i) | ||
130 | { | ||
131 | unsigned long vaddr = *position; | ||
132 | int remainder = *length; | ||
133 | |||
134 | WARN_ON(!is_vm_hugetlb_page(vma)); | ||
135 | |||
136 | while (vaddr < vma->vm_end && remainder) { | ||
137 | if (pages) { | ||
138 | pte_t *pte; | ||
139 | struct page *page; | ||
140 | |||
141 | pte = huge_pte_offset(mm, vaddr); | ||
142 | |||
143 | /* hugetlb should be locked, and hence, prefaulted */ | ||
144 | BUG_ON(!pte || pte_none(*pte)); | ||
145 | |||
146 | page = pte_page(*pte); | ||
147 | |||
148 | WARN_ON(!PageCompound(page)); | ||
149 | |||
150 | get_page(page); | ||
151 | pages[i] = page; | ||
152 | } | ||
153 | |||
154 | if (vmas) | ||
155 | vmas[i] = vma; | ||
156 | |||
157 | vaddr += PAGE_SIZE; | ||
158 | --remainder; | ||
159 | ++i; | ||
160 | } | ||
161 | |||
162 | *length = remainder; | ||
163 | *position = vaddr; | ||
164 | |||
165 | return i; | ||
166 | } | ||
167 | |||
168 | struct page *follow_huge_addr(struct mm_struct *mm, | 101 | struct page *follow_huge_addr(struct mm_struct *mm, |
169 | unsigned long address, int write) | 102 | unsigned long address, int write) |
170 | { | 103 | { |
@@ -181,84 +114,3 @@ struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address, | |||
181 | { | 114 | { |
182 | return NULL; | 115 | return NULL; |
183 | } | 116 | } |
184 | |||
185 | void unmap_hugepage_range(struct vm_area_struct *vma, | ||
186 | unsigned long start, unsigned long end) | ||
187 | { | ||
188 | struct mm_struct *mm = vma->vm_mm; | ||
189 | unsigned long address; | ||
190 | pte_t *pte; | ||
191 | struct page *page; | ||
192 | int i; | ||
193 | |||
194 | BUG_ON(start & (HPAGE_SIZE - 1)); | ||
195 | BUG_ON(end & (HPAGE_SIZE - 1)); | ||
196 | |||
197 | for (address = start; address < end; address += HPAGE_SIZE) { | ||
198 | pte = huge_pte_offset(mm, address); | ||
199 | BUG_ON(!pte); | ||
200 | if (pte_none(*pte)) | ||
201 | continue; | ||
202 | page = pte_page(*pte); | ||
203 | put_page(page); | ||
204 | for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) { | ||
205 | pte_clear(mm, address+(i*PAGE_SIZE), pte); | ||
206 | pte++; | ||
207 | } | ||
208 | } | ||
209 | add_mm_counter(mm, rss, -((end - start) >> PAGE_SHIFT)); | ||
210 | flush_tlb_range(vma, start, end); | ||
211 | } | ||
212 | |||
213 | int hugetlb_prefault(struct address_space *mapping, struct vm_area_struct *vma) | ||
214 | { | ||
215 | struct mm_struct *mm = current->mm; | ||
216 | unsigned long addr; | ||
217 | int ret = 0; | ||
218 | |||
219 | BUG_ON(vma->vm_start & ~HPAGE_MASK); | ||
220 | BUG_ON(vma->vm_end & ~HPAGE_MASK); | ||
221 | |||
222 | spin_lock(&mm->page_table_lock); | ||
223 | for (addr = vma->vm_start; addr < vma->vm_end; addr += HPAGE_SIZE) { | ||
224 | unsigned long idx; | ||
225 | pte_t *pte = huge_pte_alloc(mm, addr); | ||
226 | struct page *page; | ||
227 | |||
228 | if (!pte) { | ||
229 | ret = -ENOMEM; | ||
230 | goto out; | ||
231 | } | ||
232 | if (!pte_none(*pte)) | ||
233 | continue; | ||
234 | |||
235 | idx = ((addr - vma->vm_start) >> HPAGE_SHIFT) | ||
236 | + (vma->vm_pgoff >> (HPAGE_SHIFT - PAGE_SHIFT)); | ||
237 | page = find_get_page(mapping, idx); | ||
238 | if (!page) { | ||
239 | /* charge the fs quota first */ | ||
240 | if (hugetlb_get_quota(mapping)) { | ||
241 | ret = -ENOMEM; | ||
242 | goto out; | ||
243 | } | ||
244 | page = alloc_huge_page(); | ||
245 | if (!page) { | ||
246 | hugetlb_put_quota(mapping); | ||
247 | ret = -ENOMEM; | ||
248 | goto out; | ||
249 | } | ||
250 | ret = add_to_page_cache(page, mapping, idx, GFP_ATOMIC); | ||
251 | if (! ret) { | ||
252 | unlock_page(page); | ||
253 | } else { | ||
254 | hugetlb_put_quota(mapping); | ||
255 | free_huge_page(page); | ||
256 | goto out; | ||
257 | } | ||
258 | } | ||
259 | set_huge_pte(mm, vma, page, pte, vma->vm_flags & VM_WRITE); | ||
260 | } | ||
261 | out: | ||
262 | spin_unlock(&mm->page_table_lock); | ||
263 | return ret; | ||
264 | } | ||