1 files changed, 196 insertions, 34 deletions

diff --git a/mm/memory.c b/mm/memory.c
index a97a4cec2e1f..235ba51b2fbf 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -445,7 +445,7 @@ static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
         mm_dec_nr_pmds(tlb->mm);
 }
 
-static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
+static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
                                 unsigned long addr, unsigned long end,
                                 unsigned long floor, unsigned long ceiling)
 {
@@ -454,7 +454,7 @@ static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
         unsigned long start;
 
         start = addr;
-        pud = pud_offset(pgd, addr);
+        pud = pud_offset(p4d, addr);
         do {
                 next = pud_addr_end(addr, end);
                 if (pud_none_or_clear_bad(pud))
@@ -462,6 +462,39 @@ static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
                 free_pmd_range(tlb, pud, addr, next, floor, ceiling);
         } while (pud++, addr = next, addr != end);
 
+        start &= P4D_MASK;
+        if (start < floor)
+                return;
+        if (ceiling) {
+                ceiling &= P4D_MASK;
+                if (!ceiling)
+                        return;
+        }
+        if (end - 1 > ceiling - 1)
+                return;
+
+        pud = pud_offset(p4d, start);
+        p4d_clear(p4d);
+        pud_free_tlb(tlb, pud, start);
+}
+
+static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd,
+                                unsigned long addr, unsigned long end,
+                                unsigned long floor, unsigned long ceiling)
+{
+        p4d_t *p4d;
+        unsigned long next;
+        unsigned long start;
+
+        start = addr;
+        p4d = p4d_offset(pgd, addr);
+        do {
+                next = p4d_addr_end(addr, end);
+                if (p4d_none_or_clear_bad(p4d))
+                        continue;
+                free_pud_range(tlb, p4d, addr, next, floor, ceiling);
+        } while (p4d++, addr = next, addr != end);
+
         start &= PGDIR_MASK;
         if (start < floor)
                 return;
@@ -473,9 +506,9 @@ static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
         if (end - 1 > ceiling - 1)
                 return;
 
-        pud = pud_offset(pgd, start);
+        p4d = p4d_offset(pgd, start);
         pgd_clear(pgd);
-        pud_free_tlb(tlb, pud, start);
+        p4d_free_tlb(tlb, p4d, start);
 }
 
 /*
@@ -539,7 +572,7 @@ void free_pgd_range(struct mmu_gather *tlb,
                 next = pgd_addr_end(addr, end);
                 if (pgd_none_or_clear_bad(pgd))
                         continue;
-                free_pud_range(tlb, pgd, addr, next, floor, ceiling);
+                free_p4d_range(tlb, pgd, addr, next, floor, ceiling);
         } while (pgd++, addr = next, addr != end);
 }
 
@@ -658,7 +691,8 @@ static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
                           pte_t pte, struct page *page)
 {
         pgd_t *pgd = pgd_offset(vma->vm_mm, addr);
-        pud_t *pud = pud_offset(pgd, addr);
+        p4d_t *p4d = p4d_offset(pgd, addr);
+        pud_t *pud = pud_offset(p4d, addr);
         pmd_t *pmd = pmd_offset(pud, addr);
         struct address_space *mapping;
         pgoff_t index;
@@ -1023,16 +1057,16 @@ static inline int copy_pmd_range(struct mm_struct *dst_mm, struct mm_struct *src
 }
 
 static inline int copy_pud_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
-                pgd_t *dst_pgd, pgd_t *src_pgd, struct vm_area_struct *vma,
+                p4d_t *dst_p4d, p4d_t *src_p4d, struct vm_area_struct *vma,
                 unsigned long addr, unsigned long end)
 {
         pud_t *src_pud, *dst_pud;
         unsigned long next;
 
-        dst_pud = pud_alloc(dst_mm, dst_pgd, addr);
+        dst_pud = pud_alloc(dst_mm, dst_p4d, addr);
         if (!dst_pud)
                 return -ENOMEM;
-        src_pud = pud_offset(src_pgd, addr);
+        src_pud = pud_offset(src_p4d, addr);
         do {
                 next = pud_addr_end(addr, end);
                 if (pud_trans_huge(*src_pud) || pud_devmap(*src_pud)) {
@@ -1056,6 +1090,28 @@ static inline int copy_pud_range(struct mm_struct *dst_mm, struct mm_struct *src
         return 0;
 }
 
+static inline int copy_p4d_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
+                pgd_t *dst_pgd, pgd_t *src_pgd, struct vm_area_struct *vma,
+                unsigned long addr, unsigned long end)
+{
+        p4d_t *src_p4d, *dst_p4d;
+        unsigned long next;
+
+        dst_p4d = p4d_alloc(dst_mm, dst_pgd, addr);
+        if (!dst_p4d)
+                return -ENOMEM;
+        src_p4d = p4d_offset(src_pgd, addr);
+        do {
+                next = p4d_addr_end(addr, end);
+                if (p4d_none_or_clear_bad(src_p4d))
+                        continue;
+                if (copy_pud_range(dst_mm, src_mm, dst_p4d, src_p4d,
+                                                vma, addr, next))
+                        return -ENOMEM;
+        } while (dst_p4d++, src_p4d++, addr = next, addr != end);
+        return 0;
+}
+
 int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
                 struct vm_area_struct *vma)
 {
@@ -1111,7 +1167,7 @@ int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
                 next = pgd_addr_end(addr, end);
                 if (pgd_none_or_clear_bad(src_pgd))
                         continue;
-                if (unlikely(copy_pud_range(dst_mm, src_mm, dst_pgd, src_pgd,
+                if (unlikely(copy_p4d_range(dst_mm, src_mm, dst_pgd, src_pgd,
                                             vma, addr, next))) {
                         ret = -ENOMEM;
                         break;
@@ -1267,14 +1323,14 @@ next:
 }
 
 static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
-                                struct vm_area_struct *vma, pgd_t *pgd,
+                                struct vm_area_struct *vma, p4d_t *p4d,
                                 unsigned long addr, unsigned long end,
                                 struct zap_details *details)
 {
         pud_t *pud;
         unsigned long next;
 
-        pud = pud_offset(pgd, addr);
+        pud = pud_offset(p4d, addr);
         do {
                 next = pud_addr_end(addr, end);
                 if (pud_trans_huge(*pud) || pud_devmap(*pud)) {
@@ -1295,6 +1351,25 @@ next:
         return addr;
 }
 
+static inline unsigned long zap_p4d_range(struct mmu_gather *tlb,
+                                struct vm_area_struct *vma, pgd_t *pgd,
+                                unsigned long addr, unsigned long end,
+                                struct zap_details *details)
+{
+        p4d_t *p4d;
+        unsigned long next;
+
+        p4d = p4d_offset(pgd, addr);
+        do {
+                next = p4d_addr_end(addr, end);
+                if (p4d_none_or_clear_bad(p4d))
+                        continue;
+                next = zap_pud_range(tlb, vma, p4d, addr, next, details);
+        } while (p4d++, addr = next, addr != end);
+
+        return addr;
+}
+
 void unmap_page_range(struct mmu_gather *tlb,
                              struct vm_area_struct *vma,
                              unsigned long addr, unsigned long end,
@@ -1310,7 +1385,7 @@ void unmap_page_range(struct mmu_gather *tlb,
                 next = pgd_addr_end(addr, end);
                 if (pgd_none_or_clear_bad(pgd))
                         continue;
-                next = zap_pud_range(tlb, vma, pgd, addr, next, details);
+                next = zap_p4d_range(tlb, vma, pgd, addr, next, details);
         } while (pgd++, addr = next, addr != end);
         tlb_end_vma(tlb, vma);
 }
@@ -1465,16 +1540,24 @@ EXPORT_SYMBOL_GPL(zap_vma_ptes);
 pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
                         spinlock_t **ptl)
 {
-        pgd_t *pgd = pgd_offset(mm, addr);
-        pud_t *pud = pud_alloc(mm, pgd, addr);
-        if (pud) {
-                pmd_t *pmd = pmd_alloc(mm, pud, addr);
-                if (pmd) {
-                        VM_BUG_ON(pmd_trans_huge(*pmd));
-                        return pte_alloc_map_lock(mm, pmd, addr, ptl);
-                }
-        }
-        return NULL;
+        pgd_t *pgd;
+        p4d_t *p4d;
+        pud_t *pud;
+        pmd_t *pmd;
+
+        pgd = pgd_offset(mm, addr);
+        p4d = p4d_alloc(mm, pgd, addr);
+        if (!p4d)
+                return NULL;
+        pud = pud_alloc(mm, p4d, addr);
+        if (!pud)
+                return NULL;
+        pmd = pmd_alloc(mm, pud, addr);
+        if (!pmd)
+                return NULL;
+
+        VM_BUG_ON(pmd_trans_huge(*pmd));
+        return pte_alloc_map_lock(mm, pmd, addr, ptl);
 }
 
 /*
@@ -1740,7 +1823,7 @@ static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud,
         return 0;
 }
 
-static inline int remap_pud_range(struct mm_struct *mm, pgd_t *pgd,
+static inline int remap_pud_range(struct mm_struct *mm, p4d_t *p4d,
                         unsigned long addr, unsigned long end,
                         unsigned long pfn, pgprot_t prot)
 {
@@ -1748,7 +1831,7 @@ static inline int remap_pud_range(struct mm_struct *mm, pgd_t *pgd,
         unsigned long next;
 
         pfn -= addr >> PAGE_SHIFT;
-        pud = pud_alloc(mm, pgd, addr);
+        pud = pud_alloc(mm, p4d, addr);
         if (!pud)
                 return -ENOMEM;
         do {
@@ -1760,6 +1843,26 @@ static inline int remap_pud_range(struct mm_struct *mm, pgd_t *pgd,
         return 0;
 }
 
+static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd,
+                        unsigned long addr, unsigned long end,
+                        unsigned long pfn, pgprot_t prot)
+{
+        p4d_t *p4d;
+        unsigned long next;
+
+        pfn -= addr >> PAGE_SHIFT;
+        p4d = p4d_alloc(mm, pgd, addr);
+        if (!p4d)
+                return -ENOMEM;
+        do {
+                next = p4d_addr_end(addr, end);
+                if (remap_pud_range(mm, p4d, addr, next,
+                                pfn + (addr >> PAGE_SHIFT), prot))
+                        return -ENOMEM;
+        } while (p4d++, addr = next, addr != end);
+        return 0;
+}
+
 /**
  * remap_pfn_range - remap kernel memory to userspace
  * @vma: user vma to map to
@@ -1816,7 +1919,7 @@ int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
         flush_cache_range(vma, addr, end);
         do {
                 next = pgd_addr_end(addr, end);
-                err = remap_pud_range(mm, pgd, addr, next,
+                err = remap_p4d_range(mm, pgd, addr, next,
                                 pfn + (addr >> PAGE_SHIFT), prot);
                 if (err)
                         break;
@@ -1932,7 +2035,7 @@ static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud,
         return err;
 }
 
-static int apply_to_pud_range(struct mm_struct *mm, pgd_t *pgd,
+static int apply_to_pud_range(struct mm_struct *mm, p4d_t *p4d,
                                      unsigned long addr, unsigned long end,
                                      pte_fn_t fn, void *data)
 {
@@ -1940,7 +2043,7 @@ static int apply_to_pud_range(struct mm_struct *mm, pgd_t *pgd,
         unsigned long next;
         int err;
 
-        pud = pud_alloc(mm, pgd, addr);
+        pud = pud_alloc(mm, p4d, addr);
         if (!pud)
                 return -ENOMEM;
         do {
@@ -1952,6 +2055,26 @@ static int apply_to_pud_range(struct mm_struct *mm, pgd_t *pgd,
         return err;
 }
 
+static int apply_to_p4d_range(struct mm_struct *mm, pgd_t *pgd,
+                                     unsigned long addr, unsigned long end,
+                                     pte_fn_t fn, void *data)
+{
+        p4d_t *p4d;
+        unsigned long next;
+        int err;
+
+        p4d = p4d_alloc(mm, pgd, addr);
+        if (!p4d)
+                return -ENOMEM;
+        do {
+                next = p4d_addr_end(addr, end);
+                err = apply_to_pud_range(mm, p4d, addr, next, fn, data);
+                if (err)
+                        break;
+        } while (p4d++, addr = next, addr != end);
+        return err;
+}
+
 /*
  * Scan a region of virtual memory, filling in page tables as necessary
  * and calling a provided function on each leaf page table.
@@ -1970,7 +2093,7 @@ int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
         pgd = pgd_offset(mm, addr);
         do {
                 next = pgd_addr_end(addr, end);
-                err = apply_to_pud_range(mm, pgd, addr, next, fn, data);
+                err = apply_to_p4d_range(mm, pgd, addr, next, fn, data);
                 if (err)
                         break;
         } while (pgd++, addr = next, addr != end);
@@ -3653,11 +3776,15 @@ static int __handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
         };
         struct mm_struct *mm = vma->vm_mm;
         pgd_t *pgd;
+        p4d_t *p4d;
         int ret;
 
         pgd = pgd_offset(mm, address);
+        p4d = p4d_alloc(mm, pgd, address);
+        if (!p4d)
+                return VM_FAULT_OOM;
 
-        vmf.pud = pud_alloc(mm, pgd, address);
+        vmf.pud = pud_alloc(mm, p4d, address);
         if (!vmf.pud)
                 return VM_FAULT_OOM;
         if (pud_none(*vmf.pud) && transparent_hugepage_enabled(vma)) {
@@ -3779,12 +3906,35 @@ int handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
 }
 EXPORT_SYMBOL_GPL(handle_mm_fault);
 
+#ifndef __PAGETABLE_P4D_FOLDED
+/*
+ * Allocate p4d page table.
+ * We've already handled the fast-path in-line.
+ */
+int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
+{
+        p4d_t *new = p4d_alloc_one(mm, address);
+        if (!new)
+                return -ENOMEM;
+
+        smp_wmb(); /* See comment in __pte_alloc */
+
+        spin_lock(&mm->page_table_lock);
+        if (pgd_present(*pgd))          /* Another has populated it */
+                p4d_free(mm, new);
+        else
+                pgd_populate(mm, pgd, new);
+        spin_unlock(&mm->page_table_lock);
+        return 0;
+}
+#endif /* __PAGETABLE_P4D_FOLDED */
+
 #ifndef __PAGETABLE_PUD_FOLDED
 /*
  * Allocate page upper directory.
  * We've already handled the fast-path in-line.
  */
-int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
+int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address)
 {
         pud_t *new = pud_alloc_one(mm, address);
         if (!new)
@@ -3793,10 +3943,17 @@ int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
         smp_wmb(); /* See comment in __pte_alloc */
 
         spin_lock(&mm->page_table_lock);
-        if (pgd_present(*pgd))          /* Another has populated it */
+#ifndef __ARCH_HAS_5LEVEL_HACK
+        if (p4d_present(*p4d))          /* Another has populated it */
                 pud_free(mm, new);
         else
-                pgd_populate(mm, pgd, new);
+                p4d_populate(mm, p4d, new);
+#else
+        if (pgd_present(*p4d))          /* Another has populated it */
+                pud_free(mm, new);
+        else
+                pgd_populate(mm, p4d, new);
+#endif /* __ARCH_HAS_5LEVEL_HACK */
         spin_unlock(&mm->page_table_lock);
         return 0;
 }
@@ -3839,6 +3996,7 @@ static int __follow_pte_pmd(struct mm_struct *mm, unsigned long address,
                 pte_t **ptepp, pmd_t **pmdpp, spinlock_t **ptlp)
 {
         pgd_t *pgd;
+        p4d_t *p4d;
         pud_t *pud;
         pmd_t *pmd;
         pte_t *ptep;
@@ -3847,7 +4005,11 @@ static int __follow_pte_pmd(struct mm_struct *mm, unsigned long address,
         if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
                 goto out;
 
-        pud = pud_offset(pgd, address);
+        p4d = p4d_offset(pgd, address);
+        if (p4d_none(*p4d) || unlikely(p4d_bad(*p4d)))
+                goto out;
+
+        pud = pud_offset(p4d, address);
         if (pud_none(*pud) || unlikely(pud_bad(*pud)))
                 goto out;