8 files changed, 65 insertions, 38 deletions

diff --git a/arch/ia64/mm/hugetlbpage.c b/arch/ia64/mm/hugetlbpage.c
index 626258ae9742..df08ae7634b6 100644
--- a/arch/ia64/mm/hugetlbpage.c
+++ b/arch/ia64/mm/hugetlbpage.c
@@ -186,13 +186,30 @@ follow_huge_pmd(struct mm_struct *mm, unsigned long address, pmd_t *pmd, int wri
         return NULL;
 }
 
-/*
- * Do nothing, until we've worked out what to do!  To allow build, we
- * must remove reference to clear_page_range since it no longer exists.
- */
-void hugetlb_free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *prev,
-        unsigned long start, unsigned long end)
+void hugetlb_free_pgd_range(struct mmu_gather **tlb,
+                        unsigned long addr, unsigned long end,
+                        unsigned long floor, unsigned long ceiling)
 {
+        /*
+         * This is called only when is_hugepage_only_range(addr,),
+         * and it follows that is_hugepage_only_range(end,) also.
+         *
+         * The offset of these addresses from the base of the hugetlb
+         * region must be scaled down by HPAGE_SIZE/PAGE_SIZE so that
+         * the standard free_pgd_range will free the right page tables.
+         *
+         * If floor and ceiling are also in the hugetlb region, they
+         * must likewise be scaled down; but if outside, left unchanged.
+         */
+
+        addr = htlbpage_to_page(addr);
+        end  = htlbpage_to_page(end);
+        if (is_hugepage_only_range(tlb->mm, floor, HPAGE_SIZE))
+                floor = htlbpage_to_page(floor);
+        if (is_hugepage_only_range(tlb->mm, ceiling, HPAGE_SIZE))
+                ceiling = htlbpage_to_page(ceiling);
+
+        free_pgd_range(tlb, addr, end, floor, ceiling);
 }
 
 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
diff --git a/arch/ppc64/mm/hugetlbpage.c b/arch/ppc64/mm/hugetlbpage.c
index c62ddaff0720..8665bb57e42b 100644
--- a/arch/ppc64/mm/hugetlbpage.c
+++ b/arch/ppc64/mm/hugetlbpage.c
@@ -430,16 +430,6 @@ void unmap_hugepage_range(struct vm_area_struct *vma,
         flush_tlb_pending();
 }
 
-void hugetlb_free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *prev,
-                           unsigned long start, unsigned long end)
-{
-        /* Because the huge pgtables are only 2 level, they can take
-         * at most around 4M, much less than one hugepage which the
-         * process is presumably entitled to use.  So we don't bother
-         * freeing up the pagetables on unmap, and wait until
-         * destroy_context() to clean up the lot. */
-}
-
 int hugetlb_prefault(struct address_space *mapping, struct vm_area_struct *vma)
 {
         struct mm_struct *mm = current->mm;
diff --git a/include/asm-ia64/page.h b/include/asm-ia64/page.h
index 24aab801a8ca..08894f73abf0 100644
--- a/include/asm-ia64/page.h
+++ b/include/asm-ia64/page.h
@@ -139,7 +139,7 @@ typedef union ia64_va {
 # define HUGETLB_PAGE_ORDER     (HPAGE_SHIFT - PAGE_SHIFT)
 # define is_hugepage_only_range(mm, addr, len)          \
          (REGION_NUMBER(addr) == REGION_HPAGE &&        \
-          REGION_NUMBER((addr)+(len)) == REGION_HPAGE)
+          REGION_NUMBER((addr)+(len)-1) == REGION_HPAGE)
 extern unsigned int hpage_shift;
 #endif
 
diff --git a/include/asm-ia64/pgtable.h b/include/asm-ia64/pgtable.h
index 1757a811f436..bbf6dd757003 100644
--- a/include/asm-ia64/pgtable.h
+++ b/include/asm-ia64/pgtable.h
@@ -472,8 +472,8 @@ extern struct page *zero_page_memmap_ptr;
 #define HUGETLB_PGDIR_SIZE      (__IA64_UL(1) << HUGETLB_PGDIR_SHIFT)
 #define HUGETLB_PGDIR_MASK      (~(HUGETLB_PGDIR_SIZE-1))
 struct mmu_gather;
-extern void hugetlb_free_pgtables(struct mmu_gather *tlb,
-        struct vm_area_struct * prev, unsigned long start, unsigned long end);
+void hugetlb_free_pgd_range(struct mmu_gather **tlb, unsigned long addr,
+                unsigned long end, unsigned long floor, unsigned long ceiling);
 #endif
 
 /*
diff --git a/include/asm-ppc64/pgtable.h b/include/asm-ppc64/pgtable.h
index 4c4824653e80..33b90e2aa47d 100644
--- a/include/asm-ppc64/pgtable.h
+++ b/include/asm-ppc64/pgtable.h
@@ -500,9 +500,15 @@ extern pgd_t ioremap_dir[1024];
 
 extern void paging_init(void);
 
-struct mmu_gather;
-void hugetlb_free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *prev,
-                           unsigned long start, unsigned long end);
+/*
+ * Because the huge pgtables are only 2 level, they can take
+ * at most around 4M, much less than one hugepage which the
+ * process is presumably entitled to use.  So we don't bother
+ * freeing up the pagetables on unmap, and wait until
+ * destroy_context() to clean up the lot.
+ */
+#define hugetlb_free_pgd_range(tlb, addr, end, floor, ceiling) \
+                                                do { } while (0)
 
 /*
  * This gets called at the end of handling a page fault, when
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index ae45676d27ba..6af1ae4a8211 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -37,7 +37,8 @@ extern int sysctl_hugetlb_shm_group;
 
 #ifndef ARCH_HAS_HUGEPAGE_ONLY_RANGE
 #define is_hugepage_only_range(mm, addr, len)   0
-#define hugetlb_free_pgtables(tlb, prev, start, end) do { } while (0)
+#define hugetlb_free_pgd_range(tlb, addr, end, floor, ceiling) \
+                                                do { } while (0)
 #endif
 
 #ifndef ARCH_HAS_PREPARE_HUGEPAGE_RANGE
@@ -72,7 +73,8 @@ static inline unsigned long hugetlb_total_pages(void)
 #define prepare_hugepage_range(addr, len)       (-EINVAL)
 #define pmd_huge(x)     0
 #define is_hugepage_only_range(mm, addr, len)   0
-#define hugetlb_free_pgtables(tlb, prev, start, end) do { } while (0)
+#define hugetlb_free_pgd_range(tlb, addr, end, floor, ceiling) \
+                                                do { } while (0)
 #define alloc_huge_page()                       ({ NULL; })
 #define free_huge_page(p)                       ({ (void)(p); BUG(); })
 
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 59eca28b5ae2..c74a74ca401d 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -587,7 +587,9 @@ unsigned long unmap_vmas(struct mmu_gather **tlb, struct mm_struct *mm,
                 struct vm_area_struct *start_vma, unsigned long start_addr,
                 unsigned long end_addr, unsigned long *nr_accounted,
                 struct zap_details *);
-void free_pgtables(struct mmu_gather **tlb, struct vm_area_struct *vma,
+void free_pgd_range(struct mmu_gather **tlb, unsigned long addr,
+                unsigned long end, unsigned long floor, unsigned long ceiling);
+void free_pgtables(struct mmu_gather **tlb, struct vm_area_struct *start_vma,
                 unsigned long floor, unsigned long ceiling);
 int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
                         struct vm_area_struct *vma);
diff --git a/mm/memory.c b/mm/memory.c
index 854bd90eeca1..6bad4c4064e7 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -190,7 +190,7 @@ static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
  *
  * Must be called with pagetable lock held.
  */
-static inline void free_pgd_range(struct mmu_gather *tlb,
+void free_pgd_range(struct mmu_gather **tlb,
                         unsigned long addr, unsigned long end,
                         unsigned long floor, unsigned long ceiling)
 {
@@ -241,37 +241,47 @@ static inline void free_pgd_range(struct mmu_gather *tlb,
                 return;
 
         start = addr;
-        pgd = pgd_offset(tlb->mm, addr);
+        pgd = pgd_offset((*tlb)->mm, addr);
         do {
                 next = pgd_addr_end(addr, end);
                 if (pgd_none_or_clear_bad(pgd))
                         continue;
-                free_pud_range(tlb, pgd, addr, next, floor, ceiling);
+                free_pud_range(*tlb, pgd, addr, next, floor, ceiling);
         } while (pgd++, addr = next, addr != end);
 
-        if (!tlb_is_full_mm(tlb))
-                flush_tlb_pgtables(tlb->mm, start, end);
+        if (!tlb_is_full_mm(*tlb))
+                flush_tlb_pgtables((*tlb)->mm, start, end);
 }
 
 void free_pgtables(struct mmu_gather **tlb, struct vm_area_struct *vma,
-                                unsigned long floor, unsigned long ceiling)
+                unsigned long floor, unsigned long ceiling)
 {
         while (vma) {
                 struct vm_area_struct *next = vma->vm_next;
                 unsigned long addr = vma->vm_start;
 
-                /* Optimization: gather nearby vmas into a single call down */
-                while (next && next->vm_start <= vma->vm_end + PMD_SIZE) {
-                        vma = next;
-                        next = vma->vm_next;
-                }
-                free_pgd_range(*tlb, addr, vma->vm_end,
+                if (is_hugepage_only_range(vma->vm_mm, addr, HPAGE_SIZE)) {
+                        hugetlb_free_pgd_range(tlb, addr, vma->vm_end,
+                                floor, next? next->vm_start: ceiling);
+                } else {
+                        /*
+                         * Optimization: gather nearby vmas into one call down
+                         */
+                        while (next && next->vm_start <= vma->vm_end + PMD_SIZE
+                          && !is_hugepage_only_range(vma->vm_mm, next->vm_start,
+                                                        HPAGE_SIZE)) {
+                                vma = next;
+                                next = vma->vm_next;
+                        }
+                        free_pgd_range(tlb, addr, vma->vm_end,
                                 floor, next? next->vm_start: ceiling);
+                }
                 vma = next;
         }
 }
 
-pte_t fastcall * pte_alloc_map(struct mm_struct *mm, pmd_t *pmd, unsigned long address)
+pte_t fastcall *pte_alloc_map(struct mm_struct *mm, pmd_t *pmd,
+                                unsigned long address)
 {
         if (!pmd_present(*pmd)) {
                 struct page *new;