hugetlb: introduce pud_huge

Straight forward extensions for huge pages located in the PUD instead of
PMDs.

Signed-off-by: Andi Kleen <ak@suse.de>
Signed-off-by: Nick Piggin <npiggin@suse.de>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

2 files changed, 20 insertions, 4 deletions

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 0c74c14dd2f7..107c1ce223cb 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -1996,6 +1996,15 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
         return ret;
 }
 
+/* Can be overriden by architectures */
+__attribute__((weak)) struct page *
+follow_huge_pud(struct mm_struct *mm, unsigned long address,
+               pud_t *pud, int write)
+{
+        BUG();
+        return NULL;
+}
+
 int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
                         struct page **pages, struct vm_area_struct **vmas,
                         unsigned long *position, int *length, int i,
diff --git a/mm/memory.c b/mm/memory.c
index 02fc6b1047b0..262e3eb6601a 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -998,19 +998,24 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
                 goto no_page_table;
 
         pud = pud_offset(pgd, address);
-        if (pud_none(*pud) || unlikely(pud_bad(*pud)))
+        if (pud_none(*pud))
+                goto no_page_table;
+        if (pud_huge(*pud)) {
+                BUG_ON(flags & FOLL_GET);
+                page = follow_huge_pud(mm, address, pud, flags & FOLL_WRITE);
+                goto out;
+        }
+        if (unlikely(pud_bad(*pud)))
                 goto no_page_table;
-        
+
         pmd = pmd_offset(pud, address);
         if (pmd_none(*pmd))
                 goto no_page_table;
-
         if (pmd_huge(*pmd)) {
                 BUG_ON(flags & FOLL_GET);
                 page = follow_huge_pmd(mm, address, pmd, flags & FOLL_WRITE);
                 goto out;
         }
-
         if (unlikely(pmd_bad(*pmd)))
                 goto no_page_table;
 
@@ -1567,6 +1572,8 @@ static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud,
         unsigned long next;
         int err;
 
+        BUG_ON(pud_huge(*pud));
+
         pmd = pmd_alloc(mm, pud, addr);
         if (!pmd)
                 return -ENOMEM;