1 files changed, 65 insertions, 13 deletions

diff --git a/arch/x86/mm/hugetlbpage.c b/arch/x86/mm/hugetlbpage.c
index 0b3d567e686d..8f307d914c2e 100644
--- a/arch/x86/mm/hugetlbpage.c
+++ b/arch/x86/mm/hugetlbpage.c
@@ -124,7 +124,8 @@ int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
         return 1;
 }
 
-pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)
+pte_t *huge_pte_alloc(struct mm_struct *mm,
+                        unsigned long addr, unsigned long sz)
 {
         pgd_t *pgd;
         pud_t *pud;
@@ -133,9 +134,14 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)
         pgd = pgd_offset(mm, addr);
         pud = pud_alloc(mm, pgd, addr);
         if (pud) {
-                if (pud_none(*pud))
-                        huge_pmd_share(mm, addr, pud);
-                pte = (pte_t *) pmd_alloc(mm, pud, addr);
+                if (sz == PUD_SIZE) {
+                        pte = (pte_t *)pud;
+                } else {
+                        BUG_ON(sz != PMD_SIZE);
+                        if (pud_none(*pud))
+                                huge_pmd_share(mm, addr, pud);
+                        pte = (pte_t *) pmd_alloc(mm, pud, addr);
+                }
         }
         BUG_ON(pte && !pte_none(*pte) && !pte_huge(*pte));
 
@@ -151,8 +157,11 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
         pgd = pgd_offset(mm, addr);
         if (pgd_present(*pgd)) {
                 pud = pud_offset(pgd, addr);
-                if (pud_present(*pud))
+                if (pud_present(*pud)) {
+                        if (pud_large(*pud))
+                                return (pte_t *)pud;
                         pmd = pmd_offset(pud, addr);
+                }
         }
         return (pte_t *) pmd;
 }
@@ -188,6 +197,11 @@ int pmd_huge(pmd_t pmd)
         return 0;
 }
 
+int pud_huge(pud_t pud)
+{
+        return 0;
+}
+
 struct page *
 follow_huge_pmd(struct mm_struct *mm, unsigned long address,
                 pmd_t *pmd, int write)
@@ -208,6 +222,11 @@ int pmd_huge(pmd_t pmd)
         return !!(pmd_val(pmd) & _PAGE_PSE);
 }
 
+int pud_huge(pud_t pud)
+{
+        return !!(pud_val(pud) & _PAGE_PSE);
+}
+
 struct page *
 follow_huge_pmd(struct mm_struct *mm, unsigned long address,
                 pmd_t *pmd, int write)
@@ -216,9 +235,22 @@ follow_huge_pmd(struct mm_struct *mm, unsigned long address,
 
         page = pte_page(*(pte_t *)pmd);
         if (page)
-                page += ((address & ~HPAGE_MASK) >> PAGE_SHIFT);
+                page += ((address & ~PMD_MASK) >> PAGE_SHIFT);
+        return page;
+}
+
+struct page *
+follow_huge_pud(struct mm_struct *mm, unsigned long address,
+                pud_t *pud, int write)
+{
+        struct page *page;
+
+        page = pte_page(*(pte_t *)pud);
+        if (page)
+                page += ((address & ~PUD_MASK) >> PAGE_SHIFT);
         return page;
 }
+
 #endif
 
 /* x86_64 also uses this file */
@@ -228,6 +260,7 @@ static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file,
                 unsigned long addr, unsigned long len,
                 unsigned long pgoff, unsigned long flags)
 {
+        struct hstate *h = hstate_file(file);
         struct mm_struct *mm = current->mm;
         struct vm_area_struct *vma;
         unsigned long start_addr;
@@ -240,7 +273,7 @@ static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file,
         }
 
 full_search:
-        addr = ALIGN(start_addr, HPAGE_SIZE);
+        addr = ALIGN(start_addr, huge_page_size(h));
 
         for (vma = find_vma(mm, addr); ; vma = vma->vm_next) {
                 /* At this point:  (!vma || addr < vma->vm_end). */
@@ -262,7 +295,7 @@ full_search:
                 }
                 if (addr + mm->cached_hole_size < vma->vm_start)
                         mm->cached_hole_size = vma->vm_start - addr;
-                addr = ALIGN(vma->vm_end, HPAGE_SIZE);
+                addr = ALIGN(vma->vm_end, huge_page_size(h));
         }
 }
 
@@ -270,6 +303,7 @@ static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file,
                 unsigned long addr0, unsigned long len,
                 unsigned long pgoff, unsigned long flags)
 {
+        struct hstate *h = hstate_file(file);
         struct mm_struct *mm = current->mm;
         struct vm_area_struct *vma, *prev_vma;
         unsigned long base = mm->mmap_base, addr = addr0;
@@ -290,7 +324,7 @@ try_again:
                 goto fail;
 
         /* either no address requested or cant fit in requested address hole */
-        addr = (mm->free_area_cache - len) & HPAGE_MASK;
+        addr = (mm->free_area_cache - len) & huge_page_mask(h);
         do {
                 /*
                  * Lookup failure means no vma is above this address,
@@ -321,7 +355,7 @@ try_again:
                         largest_hole = vma->vm_start - addr;
 
                 /* try just below the current vma->vm_start */
-                addr = (vma->vm_start - len) & HPAGE_MASK;
+                addr = (vma->vm_start - len) & huge_page_mask(h);
         } while (len <= vma->vm_start);
 
 fail:
@@ -359,22 +393,23 @@ unsigned long
 hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
                 unsigned long len, unsigned long pgoff, unsigned long flags)
 {
+        struct hstate *h = hstate_file(file);
         struct mm_struct *mm = current->mm;
         struct vm_area_struct *vma;
 
-        if (len & ~HPAGE_MASK)
+        if (len & ~huge_page_mask(h))
                 return -EINVAL;
         if (len > TASK_SIZE)
                 return -ENOMEM;
 
         if (flags & MAP_FIXED) {
-                if (prepare_hugepage_range(addr, len))
+                if (prepare_hugepage_range(file, addr, len))
                         return -EINVAL;
                 return addr;
         }
 
         if (addr) {
-                addr = ALIGN(addr, HPAGE_SIZE);
+                addr = ALIGN(addr, huge_page_size(h));
                 vma = find_vma(mm, addr);
                 if (TASK_SIZE - len >= addr &&
                     (!vma || addr + len <= vma->vm_start))
@@ -390,3 +425,20 @@ hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
 
 #endif /*HAVE_ARCH_HUGETLB_UNMAPPED_AREA*/
 
+#ifdef CONFIG_X86_64
+static __init int setup_hugepagesz(char *opt)
+{
+        unsigned long ps = memparse(opt, &opt);
+        if (ps == PMD_SIZE) {
+                hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
+        } else if (ps == PUD_SIZE && cpu_has_gbpages) {
+                hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
+        } else {
+                printk(KERN_ERR "hugepagesz: Unsupported page size %lu M\n",
+                        ps >> 20);
+                return 0;
+        }
+        return 1;
+}
+__setup("hugepagesz=", setup_hugepagesz);
+#endif