powerpc: Reduce PTE table memory wastage

We allocate one page for the last level of linux page table. With THP and
large page size of 16MB, that would mean we are wasting large part
of that page. To map 16MB area, we only need a PTE space of 2K with 64K
page size. This patch reduce the space wastage by sharing the page
allocated for the last level of linux page table with multiple pmd
entries. We call these smaller chunks PTE page fragments and allocated
page, PTE page.

In order to support systems which doesn't have 64K HPTE support, we also
add another 2K to PTE page fragment. The second half of the PTE fragments
is used for storing slot and secondary bit information of an HPTE. With this
we now have a 4K PTE fragment.

We use a simple approach to share the PTE page. On allocation, we bump the
PTE page refcount to 16 and share the PTE page with the next 16 pte alloc
request. This should help in the node locality of the PTE page fragment,
assuming that the immediate pte alloc request will mostly come from the
same NUMA node. We don't try to reuse the freed PTE page fragment. Hence
we could be waisting some space.

Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Acked-by: Paul Mackerras <paulus@samba.org>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>

2 files changed, 155 insertions, 0 deletions

diff --git a/arch/powerpc/mm/mmu_context_hash64.c b/arch/powerpc/mm/mmu_context_hash64.c
index d1d1b92c5b99..178876aef40f 100644
--- a/arch/powerpc/mm/mmu_context_hash64.c
+++ b/arch/powerpc/mm/mmu_context_hash64.c
@@ -23,6 +23,7 @@
 #include <linux/slab.h>
 
 #include <asm/mmu_context.h>
+#include <asm/pgalloc.h>
 
 #include "icswx.h"
 
@@ -85,6 +86,9 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
         spin_lock_init(mm->context.cop_lockp);
 #endif /* CONFIG_PPC_ICSWX */
 
+#ifdef CONFIG_PPC_64K_PAGES
+        mm->context.pte_frag = NULL;
+#endif
         return 0;
 }
 
@@ -96,13 +100,46 @@ void __destroy_context(int context_id)
 }
 EXPORT_SYMBOL_GPL(__destroy_context);
 
+#ifdef CONFIG_PPC_64K_PAGES
+static void destroy_pagetable_page(struct mm_struct *mm)
+{
+        int count;
+        void *pte_frag;
+        struct page *page;
+
+        pte_frag = mm->context.pte_frag;
+        if (!pte_frag)
+                return;
+
+        page = virt_to_page(pte_frag);
+        /* drop all the pending references */
+        count = ((unsigned long)pte_frag & ~PAGE_MASK) >> PTE_FRAG_SIZE_SHIFT;
+        /* We allow PTE_FRAG_NR fragments from a PTE page */
+        count = atomic_sub_return(PTE_FRAG_NR - count, &page->_count);
+        if (!count) {
+                pgtable_page_dtor(page);
+                free_hot_cold_page(page, 0);
+        }
+}
+
+#else
+static inline void destroy_pagetable_page(struct mm_struct *mm)
+{
+        return;
+}
+#endif
+
+
 void destroy_context(struct mm_struct *mm)
 {
+
 #ifdef CONFIG_PPC_ICSWX
         drop_cop(mm->context.acop, mm);
         kfree(mm->context.cop_lockp);
         mm->context.cop_lockp = NULL;
 #endif /* CONFIG_PPC_ICSWX */
+
+        destroy_pagetable_page(mm);
         __destroy_context(mm->context.id);
         subpage_prot_free(mm);
         mm->context.id = MMU_NO_CONTEXT;
diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c
index 654258f165ae..a854096e1023 100644
--- a/arch/powerpc/mm/pgtable_64.c
+++ b/arch/powerpc/mm/pgtable_64.c
@@ -337,3 +337,121 @@ EXPORT_SYMBOL(__ioremap_at);
 EXPORT_SYMBOL(iounmap);
 EXPORT_SYMBOL(__iounmap);
 EXPORT_SYMBOL(__iounmap_at);
+
+#ifdef CONFIG_PPC_64K_PAGES
+static pte_t *get_from_cache(struct mm_struct *mm)
+{
+        void *pte_frag, *ret;
+
+        spin_lock(&mm->page_table_lock);
+        ret = mm->context.pte_frag;
+        if (ret) {
+                pte_frag = ret + PTE_FRAG_SIZE;
+                /*
+                 * If we have taken up all the fragments mark PTE page NULL
+                 */
+                if (((unsigned long)pte_frag & ~PAGE_MASK) == 0)
+                        pte_frag = NULL;
+                mm->context.pte_frag = pte_frag;
+        }
+        spin_unlock(&mm->page_table_lock);
+        return (pte_t *)ret;
+}
+
+static pte_t *__alloc_for_cache(struct mm_struct *mm, int kernel)
+{
+        void *ret = NULL;
+        struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK |
+                                       __GFP_REPEAT | __GFP_ZERO);
+        if (!page)
+                return NULL;
+
+        ret = page_address(page);
+        spin_lock(&mm->page_table_lock);
+        /*
+         * If we find pgtable_page set, we return
+         * the allocated page with single fragement
+         * count.
+         */
+        if (likely(!mm->context.pte_frag)) {
+                atomic_set(&page->_count, PTE_FRAG_NR);
+                mm->context.pte_frag = ret + PTE_FRAG_SIZE;
+        }
+        spin_unlock(&mm->page_table_lock);
+
+        if (!kernel)
+                pgtable_page_ctor(page);
+
+        return (pte_t *)ret;
+}
+
+pte_t *page_table_alloc(struct mm_struct *mm, unsigned long vmaddr, int kernel)
+{
+        pte_t *pte;
+
+        pte = get_from_cache(mm);
+        if (pte)
+                return pte;
+
+        return __alloc_for_cache(mm, kernel);
+}
+
+void page_table_free(struct mm_struct *mm, unsigned long *table, int kernel)
+{
+        struct page *page = virt_to_page(table);
+        if (put_page_testzero(page)) {
+                if (!kernel)
+                        pgtable_page_dtor(page);
+                free_hot_cold_page(page, 0);
+        }
+}
+
+#ifdef CONFIG_SMP
+static void page_table_free_rcu(void *table)
+{
+        struct page *page = virt_to_page(table);
+        if (put_page_testzero(page)) {
+                pgtable_page_dtor(page);
+                free_hot_cold_page(page, 0);
+        }
+}
+
+void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift)
+{
+        unsigned long pgf = (unsigned long)table;
+
+        BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
+        pgf |= shift;
+        tlb_remove_table(tlb, (void *)pgf);
+}
+
+void __tlb_remove_table(void *_table)
+{
+        void *table = (void *)((unsigned long)_table & ~MAX_PGTABLE_INDEX_SIZE);
+        unsigned shift = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE;
+
+        if (!shift)
+                /* PTE page needs special handling */
+                page_table_free_rcu(table);
+        else {
+                BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
+                kmem_cache_free(PGT_CACHE(shift), table);
+        }
+}
+#else
+void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift)
+{
+        if (!shift) {
+                /* PTE page needs special handling */
+                struct page *page = virt_to_page(table);
+                if (put_page_testzero(page)) {
+                        pgtable_page_dtor(page);
+                        free_hot_cold_page(page, 0);
+                }
+        } else {
+                BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
+                kmem_cache_free(PGT_CACHE(shift), table);
+        }
+}
+#endif
+#endif /* CONFIG_PPC_64K_PAGES */