sparc64: Eliminate PTE table memory wastage.

We've split up the PTE tables so that they take up half a page instead of
a full page.  This is in order to facilitate transparent huge page
support, which works much better if our PMDs cover 4MB instead of 8MB.

What we do is have a one-behind cache for PTE table allocations in the
mm struct.

This logic triggers only on allocations.  For example, we don't try to
keep track of free'd up page table blocks in the style that the s390 port
does.

There were only two slightly annoying aspects to this change:

1) Changing pgtable_t to be a "pte_t *".  There's all of this special
   logic in the TLB free paths that needed adjustments, as did the
   PMD populate interfaces.

2) init_new_context() needs to zap the pointer, since the mm struct
   just gets copied from the parent on fork.

Signed-off-by: David S. Miller <davem@davemloft.net>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Gerald Schaefer <gerald.schaefer@de.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

2 files changed, 110 insertions, 0 deletions

diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index 809eecf6c797..12ef4ea60c88 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -2467,3 +2467,104 @@ void __flush_tlb_all(void)
         __asm__ __volatile__("wrpr      %0, 0, %%pstate"
                              : : "r" (pstate));
 }
+
+static pte_t *get_from_cache(struct mm_struct *mm)
+{
+        struct page *page;
+        pte_t *ret;
+
+        spin_lock(&mm->page_table_lock);
+        page = mm->context.pgtable_page;
+        ret = NULL;
+        if (page) {
+                void *p = page_address(page);
+
+                mm->context.pgtable_page = NULL;
+
+                ret = (pte_t *) (p + (PAGE_SIZE / 2));
+        }
+        spin_unlock(&mm->page_table_lock);
+
+        return ret;
+}
+
+static struct page *__alloc_for_cache(struct mm_struct *mm)
+{
+        struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK |
+                                       __GFP_REPEAT | __GFP_ZERO);
+
+        if (page) {
+                spin_lock(&mm->page_table_lock);
+                if (!mm->context.pgtable_page) {
+                        atomic_set(&page->_count, 2);
+                        mm->context.pgtable_page = page;
+                }
+                spin_unlock(&mm->page_table_lock);
+        }
+        return page;
+}
+
+pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
+                            unsigned long address)
+{
+        struct page *page;
+        pte_t *pte;
+
+        pte = get_from_cache(mm);
+        if (pte)
+                return pte;
+
+        page = __alloc_for_cache(mm);
+        if (page)
+                pte = (pte_t *) page_address(page);
+
+        return pte;
+}
+
+pgtable_t pte_alloc_one(struct mm_struct *mm,
+                        unsigned long address)
+{
+        struct page *page;
+        pte_t *pte;
+
+        pte = get_from_cache(mm);
+        if (pte)
+                return pte;
+
+        page = __alloc_for_cache(mm);
+        if (page) {
+                pgtable_page_ctor(page);
+                pte = (pte_t *) page_address(page);
+        }
+
+        return pte;
+}
+
+void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
+{
+        struct page *page = virt_to_page(pte);
+        if (put_page_testzero(page))
+                free_hot_cold_page(page, 0);
+}
+
+static void __pte_free(pgtable_t pte)
+{
+        struct page *page = virt_to_page(pte);
+        if (put_page_testzero(page)) {
+                pgtable_page_dtor(page);
+                free_hot_cold_page(page, 0);
+        }
+}
+
+void pte_free(struct mm_struct *mm, pgtable_t pte)
+{
+        __pte_free(pte);
+}
+
+void pgtable_free(void *table, bool is_page)
+{
+        if (is_page)
+                __pte_free(table);
+        else
+                kmem_cache_free(pgtable_cache, table);
+}
diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c
index 70e50ea2a5b6..a35ee832baf3 100644
--- a/arch/sparc/mm/tsb.c
+++ b/arch/sparc/mm/tsb.c
@@ -445,6 +445,8 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
         mm->context.huge_pte_count = 0;
 #endif
 
+        mm->context.pgtable_page = NULL;
+
         /* copy_mm() copies over the parent's mm_struct before calling
          * us, so we need to zero out the TSB pointer or else tsb_grow()
          * will be confused and think there is an older TSB to free up.
@@ -483,10 +485,17 @@ static void tsb_destroy_one(struct tsb_config *tp)
 void destroy_context(struct mm_struct *mm)
 {
         unsigned long flags, i;
+        struct page *page;
 
         for (i = 0; i < MM_NUM_TSBS; i++)
                 tsb_destroy_one(&mm->context.tsb_block[i]);
 
+        page = mm->context.pgtable_page;
+        if (page && put_page_testzero(page)) {
+                pgtable_page_dtor(page);
+                free_hot_cold_page(page, 0);
+        }
+
         spin_lock_irqsave(&ctx_alloc_lock, flags);
 
         if (CTX_VALID(mm->context)) {