aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--arch/x86/mm/gup.c43
1 files changed, 40 insertions, 3 deletions
diff --git a/arch/x86/mm/gup.c b/arch/x86/mm/gup.c
index 6f733121f32e..3085f25b4355 100644
--- a/arch/x86/mm/gup.c
+++ b/arch/x86/mm/gup.c
@@ -124,7 +124,7 @@ static noinline int gup_huge_pmd(pmd_t pmd, unsigned long addr,
124 124
125 refs = 0; 125 refs = 0;
126 head = pte_page(pte); 126 head = pte_page(pte);
127 page = head + ((addr & ~HPAGE_MASK) >> PAGE_SHIFT); 127 page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
128 do { 128 do {
129 VM_BUG_ON(compound_head(page) != head); 129 VM_BUG_ON(compound_head(page) != head);
130 pages[*nr] = page; 130 pages[*nr] = page;
@@ -162,6 +162,38 @@ static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
162 return 1; 162 return 1;
163} 163}
164 164
165static noinline int gup_huge_pud(pud_t pud, unsigned long addr,
166 unsigned long end, int write, struct page **pages, int *nr)
167{
168 unsigned long mask;
169 pte_t pte = *(pte_t *)&pud;
170 struct page *head, *page;
171 int refs;
172
173 mask = _PAGE_PRESENT|_PAGE_USER;
174 if (write)
175 mask |= _PAGE_RW;
176 if ((pte_val(pte) & mask) != mask)
177 return 0;
178 /* hugepages are never "special" */
179 VM_BUG_ON(pte_val(pte) & _PAGE_SPECIAL);
180 VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
181
182 refs = 0;
183 head = pte_page(pte);
184 page = head + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
185 do {
186 VM_BUG_ON(compound_head(page) != head);
187 pages[*nr] = page;
188 (*nr)++;
189 page++;
190 refs++;
191 } while (addr += PAGE_SIZE, addr != end);
192 get_head_page_multiple(head, refs);
193
194 return 1;
195}
196
165static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end, 197static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end,
166 int write, struct page **pages, int *nr) 198 int write, struct page **pages, int *nr)
167{ 199{
@@ -175,8 +207,13 @@ static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end,
175 next = pud_addr_end(addr, end); 207 next = pud_addr_end(addr, end);
176 if (pud_none(pud)) 208 if (pud_none(pud))
177 return 0; 209 return 0;
178 if (!gup_pmd_range(pud, addr, next, write, pages, nr)) 210 if (unlikely(pud_large(pud))) {
179 return 0; 211 if (!gup_huge_pud(pud, addr, next, write, pages, nr))
212 return 0;
213 } else {
214 if (!gup_pmd_range(pud, addr, next, write, pages, nr))
215 return 0;
216 }
180 } while (pudp++, addr = next, addr != end); 217 } while (pudp++, addr = next, addr != end);
181 218
182 return 1; 219 return 1;