4 files changed, 269 insertions, 36 deletions

diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c
index 7370f9f33e29..266b8b2ceac9 100644
--- a/arch/powerpc/mm/hugetlbpage.c
+++ b/arch/powerpc/mm/hugetlbpage.c
@@ -30,13 +30,66 @@
 #define NUM_LOW_AREAS   (0x100000000UL >> SID_SHIFT)
 #define NUM_HIGH_AREAS  (PGTABLE_RANGE >> HTLB_AREA_SHIFT)
 
+#ifdef CONFIG_PPC_64K_PAGES
+#define HUGEPTE_INDEX_SIZE      (PMD_SHIFT-HPAGE_SHIFT)
+#else
+#define HUGEPTE_INDEX_SIZE      (PUD_SHIFT-HPAGE_SHIFT)
+#endif
+#define PTRS_PER_HUGEPTE        (1 << HUGEPTE_INDEX_SIZE)
+#define HUGEPTE_TABLE_SIZE      (sizeof(pte_t) << HUGEPTE_INDEX_SIZE)
+
+#define HUGEPD_SHIFT            (HPAGE_SHIFT + HUGEPTE_INDEX_SIZE)
+#define HUGEPD_SIZE             (1UL << HUGEPD_SHIFT)
+#define HUGEPD_MASK             (~(HUGEPD_SIZE-1))
+
+#define huge_pgtable_cache      (pgtable_cache[HUGEPTE_CACHE_NUM])
+
+/* Flag to mark huge PD pointers.  This means pmd_bad() and pud_bad()
+ * will choke on pointers to hugepte tables, which is handy for
+ * catching screwups early. */
+#define HUGEPD_OK       0x1
+
+typedef struct { unsigned long pd; } hugepd_t;
+
+#define hugepd_none(hpd)        ((hpd).pd == 0)
+
+static inline pte_t *hugepd_page(hugepd_t hpd)
+{
+        BUG_ON(!(hpd.pd & HUGEPD_OK));
+        return (pte_t *)(hpd.pd & ~HUGEPD_OK);
+}
+
+static inline pte_t *hugepte_offset(hugepd_t *hpdp, unsigned long addr)
+{
+        unsigned long idx = ((addr >> HPAGE_SHIFT) & (PTRS_PER_HUGEPTE-1));
+        pte_t *dir = hugepd_page(*hpdp);
+
+        return dir + idx;
+}
+
+static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
+                           unsigned long address)
+{
+        pte_t *new = kmem_cache_alloc(huge_pgtable_cache,
+                                      GFP_KERNEL|__GFP_REPEAT);
+
+        if (! new)
+                return -ENOMEM;
+
+        spin_lock(&mm->page_table_lock);
+        if (!hugepd_none(*hpdp))
+                kmem_cache_free(huge_pgtable_cache, new);
+        else
+                hpdp->pd = (unsigned long)new | HUGEPD_OK;
+        spin_unlock(&mm->page_table_lock);
+        return 0;
+}
+
 /* Modelled after find_linux_pte() */
 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
 {
         pgd_t *pg;
         pud_t *pu;
-        pmd_t *pm;
-        pte_t *pt;
 
         BUG_ON(! in_hugepage_area(mm->context, addr));
 
@@ -46,26 +99,14 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
         if (!pgd_none(*pg)) {
                 pu = pud_offset(pg, addr);
                 if (!pud_none(*pu)) {
-                        pm = pmd_offset(pu, addr);
 #ifdef CONFIG_PPC_64K_PAGES
-                        /* Currently, we use the normal PTE offset within full
-                         * size PTE pages, thus our huge PTEs are scattered in
-                         * the PTE page and we do waste some. We may change
-                         * that in the future, but the current mecanism keeps
-                         * things much simpler
-                         */
-                        if (!pmd_none(*pm)) {
-                                /* Note: pte_offset_* are all equivalent on
-                                 * ppc64 as we don't have HIGHMEM
-                                 */
-                                pt = pte_offset_kernel(pm, addr);
-                                return pt;
-                        }
-#else /* CONFIG_PPC_64K_PAGES */
-                        /* On 4k pages, we put huge PTEs in the PMD page */
-                        pt = (pte_t *)pm;
-                        return pt;
-#endif /* CONFIG_PPC_64K_PAGES */
+                        pmd_t *pm;
+                        pm = pmd_offset(pu, addr);
+                        if (!pmd_none(*pm))
+                                return hugepte_offset((hugepd_t *)pm, addr);
+#else
+                        return hugepte_offset((hugepd_t *)pu, addr);
+#endif
                 }
         }
 
@@ -76,8 +117,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)
 {
         pgd_t *pg;
         pud_t *pu;
-        pmd_t *pm;
-        pte_t *pt;
+        hugepd_t *hpdp = NULL;
 
         BUG_ON(! in_hugepage_area(mm->context, addr));
 
@@ -87,23 +127,182 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)
         pu = pud_alloc(mm, pg, addr);
 
         if (pu) {
+#ifdef CONFIG_PPC_64K_PAGES
+                pmd_t *pm;
                 pm = pmd_alloc(mm, pu, addr);
-                if (pm) {
+                if (pm)
+                        hpdp = (hugepd_t *)pm;
+#else
+                hpdp = (hugepd_t *)pu;
+#endif
+        }
+
+        if (! hpdp)
+                return NULL;
+
+        if (hugepd_none(*hpdp) && __hugepte_alloc(mm, hpdp, addr))
+                return NULL;
+
+        return hugepte_offset(hpdp, addr);
+}
+
+static void free_hugepte_range(struct mmu_gather *tlb, hugepd_t *hpdp)
+{
+        pte_t *hugepte = hugepd_page(*hpdp);
+
+        hpdp->pd = 0;
+        tlb->need_flush = 1;
+        pgtable_free_tlb(tlb, pgtable_free_cache(hugepte, HUGEPTE_CACHE_NUM,
+                                                 HUGEPTE_TABLE_SIZE-1));
+}
+
 #ifdef CONFIG_PPC_64K_PAGES
-                        /* See comment in huge_pte_offset. Note that if we ever
-                         * want to put the page size in the PMD, we would have
-                         * to open code our own pte_alloc* function in order
-                         * to populate and set the size atomically
-                         */
-                        pt = pte_alloc_map(mm, pm, addr);
-#else /* CONFIG_PPC_64K_PAGES */
-                        pt = (pte_t *)pm;
-#endif /* CONFIG_PPC_64K_PAGES */
-                        return pt;
-                }
+static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
+                                   unsigned long addr, unsigned long end,
+                                   unsigned long floor, unsigned long ceiling)
+{
+        pmd_t *pmd;
+        unsigned long next;
+        unsigned long start;
+
+        start = addr;
+        pmd = pmd_offset(pud, addr);
+        do {
+                next = pmd_addr_end(addr, end);
+                if (pmd_none(*pmd))
+                        continue;
+                free_hugepte_range(tlb, (hugepd_t *)pmd);
+        } while (pmd++, addr = next, addr != end);
+
+        start &= PUD_MASK;
+        if (start < floor)
+                return;
+        if (ceiling) {
+                ceiling &= PUD_MASK;
+                if (!ceiling)
+                        return;
         }
+        if (end - 1 > ceiling - 1)
+                return;
 
-        return NULL;
+        pmd = pmd_offset(pud, start);
+        pud_clear(pud);
+        pmd_free_tlb(tlb, pmd);
+}
+#endif
+
+static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
+                                   unsigned long addr, unsigned long end,
+                                   unsigned long floor, unsigned long ceiling)
+{
+        pud_t *pud;
+        unsigned long next;
+        unsigned long start;
+
+        start = addr;
+        pud = pud_offset(pgd, addr);
+        do {
+                next = pud_addr_end(addr, end);
+#ifdef CONFIG_PPC_64K_PAGES
+                if (pud_none_or_clear_bad(pud))
+                        continue;
+                hugetlb_free_pmd_range(tlb, pud, addr, next, floor, ceiling);
+#else
+                if (pud_none(*pud))
+                        continue;
+                free_hugepte_range(tlb, (hugepd_t *)pud);
+#endif
+        } while (pud++, addr = next, addr != end);
+
+        start &= PGDIR_MASK;
+        if (start < floor)
+                return;
+        if (ceiling) {
+                ceiling &= PGDIR_MASK;
+                if (!ceiling)
+                        return;
+        }
+        if (end - 1 > ceiling - 1)
+                return;
+
+        pud = pud_offset(pgd, start);
+        pgd_clear(pgd);
+        pud_free_tlb(tlb, pud);
+}
+
+/*
+ * This function frees user-level page tables of a process.
+ *
+ * Must be called with pagetable lock held.
+ */
+void hugetlb_free_pgd_range(struct mmu_gather **tlb,
+                            unsigned long addr, unsigned long end,
+                            unsigned long floor, unsigned long ceiling)
+{
+        pgd_t *pgd;
+        unsigned long next;
+        unsigned long start;
+
+        /*
+         * Comments below take from the normal free_pgd_range().  They
+         * apply here too.  The tests against HUGEPD_MASK below are
+         * essential, because we *don't* test for this at the bottom
+         * level.  Without them we'll attempt to free a hugepte table
+         * when we unmap just part of it, even if there are other
+         * active mappings using it.
+         *
+         * The next few lines have given us lots of grief...
+         *
+         * Why are we testing HUGEPD* at this top level?  Because
+         * often there will be no work to do at all, and we'd prefer
+         * not to go all the way down to the bottom just to discover
+         * that.
+         *
+         * Why all these "- 1"s?  Because 0 represents both the bottom
+         * of the address space and the top of it (using -1 for the
+         * top wouldn't help much: the masks would do the wrong thing).
+         * The rule is that addr 0 and floor 0 refer to the bottom of
+         * the address space, but end 0 and ceiling 0 refer to the top
+         * Comparisons need to use "end - 1" and "ceiling - 1" (though
+         * that end 0 case should be mythical).
+         *
+         * Wherever addr is brought up or ceiling brought down, we
+         * must be careful to reject "the opposite 0" before it
+         * confuses the subsequent tests.  But what about where end is
+         * brought down by HUGEPD_SIZE below? no, end can't go down to
+         * 0 there.
+         *
+         * Whereas we round start (addr) and ceiling down, by different
+         * masks at different levels, in order to test whether a table
+         * now has no other vmas using it, so can be freed, we don't
+         * bother to round floor or end up - the tests don't need that.
+         */
+
+        addr &= HUGEPD_MASK;
+        if (addr < floor) {
+                addr += HUGEPD_SIZE;
+                if (!addr)
+                        return;
+        }
+        if (ceiling) {
+                ceiling &= HUGEPD_MASK;
+                if (!ceiling)
+                        return;
+        }
+        if (end - 1 > ceiling - 1)
+                end -= HUGEPD_SIZE;
+        if (addr > end - 1)
+                return;
+
+        start = addr;
+        pgd = pgd_offset((*tlb)->mm, addr);
+        do {
+                BUG_ON(! in_hugepage_area((*tlb)->mm->context, addr));
+                next = pgd_addr_end(addr, end);
+                if (pgd_none_or_clear_bad(pgd))
+                        continue;
+                hugetlb_free_pud_range(*tlb, pgd, addr, next, floor, ceiling);
+        } while (pgd++, addr = next, addr != end);
 }
 
 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
@@ -841,3 +1040,27 @@ repeat:
  out:
         return err;
 }
+
+static void zero_ctor(void *addr, kmem_cache_t *cache, unsigned long flags)
+{
+        memset(addr, 0, kmem_cache_size(cache));
+}
+
+static int __init hugetlbpage_init(void)
+{
+        if (!cpu_has_feature(CPU_FTR_16M_PAGE))
+                return -ENODEV;
+
+        huge_pgtable_cache = kmem_cache_create("hugepte_cache",
+                                               HUGEPTE_TABLE_SIZE,
+                                               HUGEPTE_TABLE_SIZE,
+                                               SLAB_HWCACHE_ALIGN |
+                                               SLAB_MUST_HWCACHE_ALIGN,
+                                               zero_ctor, NULL);
+        if (! huge_pgtable_cache)
+                panic("hugetlbpage_init(): could not create hugepte cache\n");
+
+        return 0;
+}
+
+module_init(hugetlbpage_init);
diff --git a/arch/powerpc/mm/init_64.c b/arch/powerpc/mm/init_64.c
index babebd15bdc4..9e30f968c184 100644
--- a/arch/powerpc/mm/init_64.c
+++ b/arch/powerpc/mm/init_64.c
@@ -162,7 +162,14 @@ static const char *pgtable_cache_name[ARRAY_SIZE(pgtable_cache_size)] = {
 };
 #endif /* CONFIG_PPC_64K_PAGES */
 
+#ifdef CONFIG_HUGETLB_PAGE
+/* Hugepages need one extra cache, initialized in hugetlbpage.c.  We
+ * can't put into the tables above, because HPAGE_SHIFT is not compile
+ * time constant. */
+kmem_cache_t *pgtable_cache[ARRAY_SIZE(pgtable_cache_size)+1];
+#else
 kmem_cache_t *pgtable_cache[ARRAY_SIZE(pgtable_cache_size)];
+#endif
 
 void pgtable_cache_init(void)
 {
diff --git a/include/asm-powerpc/page_64.h b/include/asm-powerpc/page_64.h
index 3fb061bab9ec..eab779c21995 100644
--- a/include/asm-powerpc/page_64.h
+++ b/include/asm-powerpc/page_64.h
@@ -101,6 +101,7 @@ extern unsigned int HPAGE_SHIFT;
                                       - (1U << GET_HTLB_AREA(addr))) & 0xffff)
 
 #define ARCH_HAS_HUGEPAGE_ONLY_RANGE
+#define ARCH_HAS_HUGETLB_FREE_PGD_RANGE
 #define ARCH_HAS_PREPARE_HUGEPAGE_RANGE
 #define ARCH_HAS_SETCLEAR_HUGE_PTE
 
diff --git a/include/asm-powerpc/pgalloc.h b/include/asm-powerpc/pgalloc.h
index a00ee002cd11..9f0917c68659 100644
--- a/include/asm-powerpc/pgalloc.h
+++ b/include/asm-powerpc/pgalloc.h
@@ -17,11 +17,13 @@ extern kmem_cache_t *pgtable_cache[];
 #define PTE_CACHE_NUM   0
 #define PMD_CACHE_NUM   1
 #define PGD_CACHE_NUM   2
+#define HUGEPTE_CACHE_NUM 3
 #else
 #define PTE_CACHE_NUM   0
 #define PMD_CACHE_NUM   1
 #define PUD_CACHE_NUM   1
 #define PGD_CACHE_NUM   0
+#define HUGEPTE_CACHE_NUM 2
 #endif
 
 /*