1 files changed, 60 insertions, 21 deletions

diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index b4024d688f38..0dd80222b20b 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -86,12 +86,12 @@ static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end)
         } while (pmd++, addr = next, addr != end);
 }
 
-static void vunmap_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end)
+static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end)
 {
         pud_t *pud;
         unsigned long next;
 
-        pud = pud_offset(pgd, addr);
+        pud = pud_offset(p4d, addr);
         do {
                 next = pud_addr_end(addr, end);
                 if (pud_clear_huge(pud))
@@ -102,6 +102,22 @@ static void vunmap_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end)
         } while (pud++, addr = next, addr != end);
 }
 
+static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end)
+{
+        p4d_t *p4d;
+        unsigned long next;
+
+        p4d = p4d_offset(pgd, addr);
+        do {
+                next = p4d_addr_end(addr, end);
+                if (p4d_clear_huge(p4d))
+                        continue;
+                if (p4d_none_or_clear_bad(p4d))
+                        continue;
+                vunmap_pud_range(p4d, addr, next);
+        } while (p4d++, addr = next, addr != end);
+}
+
 static void vunmap_page_range(unsigned long addr, unsigned long end)
 {
         pgd_t *pgd;
@@ -113,7 +129,7 @@ static void vunmap_page_range(unsigned long addr, unsigned long end)
                 next = pgd_addr_end(addr, end);
                 if (pgd_none_or_clear_bad(pgd))
                         continue;
-                vunmap_pud_range(pgd, addr, next);
+                vunmap_p4d_range(pgd, addr, next);
         } while (pgd++, addr = next, addr != end);
 }
 
@@ -160,13 +176,13 @@ static int vmap_pmd_range(pud_t *pud, unsigned long addr,
         return 0;
 }
 
-static int vmap_pud_range(pgd_t *pgd, unsigned long addr,
+static int vmap_pud_range(p4d_t *p4d, unsigned long addr,
                 unsigned long end, pgprot_t prot, struct page **pages, int *nr)
 {
         pud_t *pud;
         unsigned long next;
 
-        pud = pud_alloc(&init_mm, pgd, addr);
+        pud = pud_alloc(&init_mm, p4d, addr);
         if (!pud)
                 return -ENOMEM;
         do {
@@ -177,6 +193,23 @@ static int vmap_pud_range(pgd_t *pgd, unsigned long addr,
         return 0;
 }
 
+static int vmap_p4d_range(pgd_t *pgd, unsigned long addr,
+                unsigned long end, pgprot_t prot, struct page **pages, int *nr)
+{
+        p4d_t *p4d;
+        unsigned long next;
+
+        p4d = p4d_alloc(&init_mm, pgd, addr);
+        if (!p4d)
+                return -ENOMEM;
+        do {
+                next = p4d_addr_end(addr, end);
+                if (vmap_pud_range(p4d, addr, next, prot, pages, nr))
+                        return -ENOMEM;
+        } while (p4d++, addr = next, addr != end);
+        return 0;
+}
+
 /*
  * Set up page tables in kva (addr, end). The ptes shall have prot "prot", and
  * will have pfns corresponding to the "pages" array.
@@ -196,7 +229,7 @@ static int vmap_page_range_noflush(unsigned long start, unsigned long end,
         pgd = pgd_offset_k(addr);
         do {
                 next = pgd_addr_end(addr, end);
-                err = vmap_pud_range(pgd, addr, next, prot, pages, &nr);
+                err = vmap_p4d_range(pgd, addr, next, prot, pages, &nr);
                 if (err)
                         return err;
         } while (pgd++, addr = next, addr != end);
@@ -237,6 +270,10 @@ struct page *vmalloc_to_page(const void *vmalloc_addr)
         unsigned long addr = (unsigned long) vmalloc_addr;
         struct page *page = NULL;
         pgd_t *pgd = pgd_offset_k(addr);
+        p4d_t *p4d;
+        pud_t *pud;
+        pmd_t *pmd;
+        pte_t *ptep, pte;
 
         /*
          * XXX we might need to change this if we add VIRTUAL_BUG_ON for
@@ -244,21 +281,23 @@ struct page *vmalloc_to_page(const void *vmalloc_addr)
          */
         VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr));
 
-        if (!pgd_none(*pgd)) {
-                pud_t *pud = pud_offset(pgd, addr);
-                if (!pud_none(*pud)) {
-                        pmd_t *pmd = pmd_offset(pud, addr);
-                        if (!pmd_none(*pmd)) {
-                                pte_t *ptep, pte;
-
-                                ptep = pte_offset_map(pmd, addr);
-                                pte = *ptep;
-                                if (pte_present(pte))
-                                        page = pte_page(pte);
-                                pte_unmap(ptep);
-                        }
-                }
-        }
+        if (pgd_none(*pgd))
+                return NULL;
+        p4d = p4d_offset(pgd, addr);
+        if (p4d_none(*p4d))
+                return NULL;
+        pud = pud_offset(p4d, addr);
+        if (pud_none(*pud))
+                return NULL;
+        pmd = pmd_offset(pud, addr);
+        if (pmd_none(*pmd))
+                return NULL;
+
+        ptep = pte_offset_map(pmd, addr);
+        pte = *ptep;
+        if (pte_present(pte))
+                page = pte_page(pte);
+        pte_unmap(ptep);
         return page;
 }
 EXPORT_SYMBOL(vmalloc_to_page);