1 files changed, 40 insertions, 6 deletions

diff --git a/mm/gup.c b/mm/gup.c
index 9c047e951aa3..c74bad1bf6e8 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -226,6 +226,7 @@ struct page *follow_page_mask(struct vm_area_struct *vma,
                               unsigned int *page_mask)
 {
         pgd_t *pgd;
+        p4d_t *p4d;
         pud_t *pud;
         pmd_t *pmd;
         spinlock_t *ptl;
@@ -243,8 +244,13 @@ struct page *follow_page_mask(struct vm_area_struct *vma,
         pgd = pgd_offset(mm, address);
         if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
                 return no_page_table(vma, flags);
-
-        pud = pud_offset(pgd, address);
+        p4d = p4d_offset(pgd, address);
+        if (p4d_none(*p4d))
+                return no_page_table(vma, flags);
+        BUILD_BUG_ON(p4d_huge(*p4d));
+        if (unlikely(p4d_bad(*p4d)))
+                return no_page_table(vma, flags);
+        pud = pud_offset(p4d, address);
         if (pud_none(*pud))
                 return no_page_table(vma, flags);
         if (pud_huge(*pud) && vma->vm_flags & VM_HUGETLB) {
@@ -325,6 +331,7 @@ static int get_gate_page(struct mm_struct *mm, unsigned long address,
                 struct page **page)
 {
         pgd_t *pgd;
+        p4d_t *p4d;
         pud_t *pud;
         pmd_t *pmd;
         pte_t *pte;
@@ -338,7 +345,9 @@ static int get_gate_page(struct mm_struct *mm, unsigned long address,
         else
                 pgd = pgd_offset_gate(mm, address);
         BUG_ON(pgd_none(*pgd));
-        pud = pud_offset(pgd, address);
+        p4d = p4d_offset(pgd, address);
+        BUG_ON(p4d_none(*p4d));
+        pud = pud_offset(p4d, address);
         BUG_ON(pud_none(*pud));
         pmd = pmd_offset(pud, address);
         if (pmd_none(*pmd))
@@ -1400,13 +1409,13 @@ static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
         return 1;
 }
 
-static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end,
+static int gup_pud_range(p4d_t p4d, unsigned long addr, unsigned long end,
                          int write, struct page **pages, int *nr)
 {
         unsigned long next;
         pud_t *pudp;
 
-        pudp = pud_offset(&pgd, addr);
+        pudp = pud_offset(&p4d, addr);
         do {
                 pud_t pud = READ_ONCE(*pudp);
 
@@ -1428,6 +1437,31 @@ static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end,
         return 1;
 }
 
+static int gup_p4d_range(pgd_t pgd, unsigned long addr, unsigned long end,
+                         int write, struct page **pages, int *nr)
+{
+        unsigned long next;
+        p4d_t *p4dp;
+
+        p4dp = p4d_offset(&pgd, addr);
+        do {
+                p4d_t p4d = READ_ONCE(*p4dp);
+
+                next = p4d_addr_end(addr, end);
+                if (p4d_none(p4d))
+                        return 0;
+                BUILD_BUG_ON(p4d_huge(p4d));
+                if (unlikely(is_hugepd(__hugepd(p4d_val(p4d))))) {
+                        if (!gup_huge_pd(__hugepd(p4d_val(p4d)), addr,
+                                         P4D_SHIFT, next, write, pages, nr))
+                                return 0;
+                } else if (!gup_p4d_range(p4d, addr, next, write, pages, nr))
+                        return 0;
+        } while (p4dp++, addr = next, addr != end);
+
+        return 1;
+}
+
 /*
  * Like get_user_pages_fast() except it's IRQ-safe in that it won't fall back to
  * the regular GUP. It will only return non-negative values.
@@ -1478,7 +1512,7 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
                         if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr,
                                          PGDIR_SHIFT, next, write, pages, &nr))
                                 break;
-                } else if (!gup_pud_range(pgd, addr, next, write, pages, &nr))
+                } else if (!gup_p4d_range(pgd, addr, next, write, pages, &nr))
                         break;
         } while (pgdp++, addr = next, addr != end);
         local_irq_restore(flags);